Merge "Combine bridge networks"

This commit is contained in:
Jenkins 2017-01-10 01:56:55 +00:00 committed by Gerrit Code Review
commit 25a852d100
13 changed files with 703 additions and 599 deletions

View File

@ -70,6 +70,7 @@ TRICIRCLE_NEUTRON_PORT=20001
[DEFAULT]
core_plugin=tricircle.network.local_plugin.TricirclePlugin
service_plugins=tricircle.network.local_l3_plugin.TricircleL3Plugin
[client]
admin_username=admin

View File

@ -0,0 +1,4 @@
---
features:
- North-south bridge network and east-west bridge network are combined into
one to bring better DVR and shared VxLAN network support.

View File

@ -64,12 +64,9 @@ client_opts = [
default='Default',
help='tenant domain name of admin account, needed when'
' auto_refresh_endpoint set to True'),
cfg.StrOpt('ew_bridge_cidr',
cfg.StrOpt('bridge_cidr',
default='100.0.0.0/9',
help='cidr pool of the east-west bridge network'),
cfg.StrOpt('ns_bridge_cidr',
default='100.128.0.0/9',
help='cidr pool of the north-south bridge network')
help='cidr pool of the bridge network')
]
client_opt_group = cfg.OptGroup('client')
cfg.CONF.register_group(client_opt_group)

View File

@ -30,9 +30,13 @@ RT_VOl_METADATA = 'volume_metadata'
RT_BACKUP = 'backup'
RT_SNAPSHOT = 'snapshot'
RT_NETWORK = 'network'
RT_SD_NETWORK = 'shadow_network'
RT_SUBNET = 'subnet'
RT_SD_SUBNET = 'shadow_subnet'
RT_PORT = 'port'
RT_SD_PORT = 'shadow_port'
RT_ROUTER = 'router'
RT_NS_ROUTER = 'ns_router'
RT_SG = 'security_group'
@ -52,23 +56,23 @@ R_LIBERTY = 'liberty'
R_MITAKA = 'mitaka'
# l3 bridge networking elements
ew_bridge_subnet_pool_name = 'ew_bridge_subnet_pool'
ew_bridge_net_name = 'ew_bridge_net_%s' # project_id
ew_bridge_subnet_name = 'ew_bridge_subnet_%s' # project_id
ew_bridge_port_name = 'ew_bridge_port_%s_%s' # project_id b_router_id
bridge_subnet_pool_name = 'bridge_subnet_pool'
bridge_net_name = 'bridge_net_%s' # project_id
bridge_subnet_name = 'bridge_subnet_%s' # project_id
bridge_port_name = 'bridge_port_%s_%s' # project_id b_router_id
ns_bridge_subnet_pool_name = 'ns_bridge_subnet_pool'
ns_bridge_net_name = 'ns_bridge_net_%s' # project_id
ns_bridge_subnet_name = 'ns_bridge_subnet_%s' # project_id
# for external gateway port: project_id b_router_id None
# for floating ip port: project_id None b_internal_port_id
ns_bridge_port_name = 'ns_bridge_port_%s_%s_%s'
ns_router_name = 'ns_router_%s'
shadow_port_name = 'shadow_port_%s'
dhcp_port_name = 'dhcp_port_%s' # subnet_id
interface_port_name = 'interface_%s_%s' # b_pod_id t_subnet_id
interface_port_device_id = 'reserved_gateway_port'
MAX_INT = 0x7FFFFFFF
DEFAULT_DESTINATION = '0.0.0.0/0'
expire_time = datetime.datetime(2000, 1, 1)
# job status

View File

@ -40,7 +40,9 @@ def get_or_create_route(t_ctx, q_ctx,
t_ctx, models.ResourceRouting,
[{'key': 'top_id', 'comparator': 'eq', 'value': _id},
{'key': 'pod_id', 'comparator': 'eq',
'value': pod['pod_id']}], [])
'value': pod['pod_id']},
{'key': 'resource_type', 'comparator': 'eq',
'value': _type}], [])
if routes:
route = routes[0]
if route['bottom_id']:
@ -109,7 +111,7 @@ def get_or_create_element(t_ctx, q_ctx,
core.delete_resource(t_ctx,
models.ResourceRouting,
route['id'])
except db_exc.ResourceNotFound:
except db_exc.ColumnError:
# NOTE(zhiyuan) this is a rare case that other worker
# considers the route expires and delete it though it
# was just created, maybe caused by out-of-sync time

View File

@ -60,8 +60,8 @@ class ResourceRouting(core.ModelBase, core.DictBase, models.TimestampMixin):
__tablename__ = 'resource_routings'
__table_args__ = (
schema.UniqueConstraint(
'top_id', 'pod_id',
name='resource_routings0top_id0pod_id'),
'top_id', 'pod_id', 'resource_type',
name='resource_routings0top_id0pod_id0resource_type'),
)
attributes = ['id', 'top_id', 'bottom_id', 'pod_id', 'project_id',
'resource_type', 'created_at', 'updated_at']

View File

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import copy
from oslo_config import cfg
@ -274,11 +275,22 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
try:
mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, network_id, t_constants.RT_NETWORK)
mappings.extend(db_api.get_bottom_mappings_by_top_id(
t_ctx, network_id, t_constants.RT_SD_NETWORK))
processed_pod_set = set()
for mapping in mappings:
region_name = mapping[0]['region_name']
if region_name in processed_pod_set:
continue
processed_pod_set.add(region_name)
bottom_network_id = mapping[1]
self._get_client(region_name).delete_networks(
t_ctx, bottom_network_id)
# we do not specify resource_type when deleting routing entries
# so if both "network" and "shadow_network" type entries exist
# in one pod(this is possible for cross-pod network), we delete
# them at the same time
with t_ctx.session.begin():
core.delete_resources(
t_ctx, models.ResourceRouting,
@ -386,14 +398,25 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
try:
mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, subnet_id, t_constants.RT_SUBNET)
mappings.extend(db_api.get_bottom_mappings_by_top_id(
t_ctx, subnet_id, t_constants.RT_SD_SUBNET))
processed_pod_set = set()
for mapping in mappings:
region_name = mapping[0]['region_name']
if region_name in processed_pod_set:
continue
processed_pod_set.add(region_name)
bottom_subnet_id = mapping[1]
self._get_client(region_name).delete_subnets(
t_ctx, bottom_subnet_id)
interface_name = t_constants.interface_port_name % (
mapping[0]['region_name'], subnet_id)
self._delete_pre_created_port(t_ctx, context, interface_name)
# we do not specify resource_type when deleting routing entries
# so if both "subnet" and "shadow_subnet" type entries exist in
# one pod(this is possible for cross-pod network), we delete
# them at the same time
with t_ctx.session.begin():
core.delete_resources(
t_ctx, models.ResourceRouting,
@ -473,6 +496,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
self.xjob_handler.setup_bottom_router(
admin_context, res['network_id'],
interfaces[0]['device_id'], pod['pod_id'])
else:
LOG.debug('Update port: no interfaces found, xjob not'
'triggered')
self.xjob_handler.configure_security_group_rules(t_ctx,
res['tenant_id'])
@ -661,10 +687,18 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
id_list.append(_id)
return id_list
@staticmethod
def _filter_shadow_port(ports, pod_id, port_pod_map):
port_list = []
for port in ports:
if pod_id not in port_pod_map[port['id']]:
port_list.append(port)
return port_list
def _get_ports_from_pod_with_number(self, context,
current_pod, number, last_port_id,
bottom_top_map, top_bottom_map,
filters=None):
port_pod_map, filters=None):
# NOTE(zhiyuan) last_port_id is top id, also id in returned port dict
# also uses top id. when interacting with bottom pod, need to map
# top to bottom in request and map bottom to top in response
@ -678,7 +712,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
for key, value in _filters:
if key == 'fixed_ips':
if 'ip_address' in value:
_filters[key] = 'ip_address=%s' % value['ip_address']
_filters[key] = 'ip_address=%s' % value[
'ip_address'][0]
continue
id_list = self._get_map_filter_ids(
key, value, current_pod['pod_id'], top_bottom_map)
@ -689,8 +724,10 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
# map top id to bottom id in request
params['marker'] = top_bottom_map[last_port_id]
res = q_client.get(q_client.ports_path, params=params)
ports = self._filter_shadow_port(res['ports'], current_pod['pod_id'],
port_pod_map)
# map bottom id to top id in client response
mapped_port_list = self._map_ports_from_bottom_to_top(res['ports'],
mapped_port_list = self._map_ports_from_bottom_to_top(ports,
bottom_top_map)
del res['ports']
res['ports'] = mapped_port_list
@ -712,7 +749,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
# need to map
next_res = self._get_ports_from_pod_with_number(
context, next_pod, number - len(res['ports']), '',
bottom_top_map, top_bottom_map, filters)
bottom_top_map, top_bottom_map, port_pod_map, filters)
next_res['ports'].extend(res['ports'])
return next_res
@ -780,6 +817,16 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
key = '%s_%s' % (route['pod_id'], route['top_id'])
top_bottom_map[key] = route['bottom_id']
port_pod_map = collections.defaultdict(set)
route_filters = [{'key': 'resource_type',
'comparator': 'eq',
'value': t_constants.RT_SD_PORT}]
routes = core.query_resource(t_ctx, models.ResourceRouting,
route_filters, [])
for route in routes:
if route['bottom_id']:
port_pod_map[route['bottom_id']].add(route['pod_id'])
if limit:
if marker:
mappings = db_api.get_bottom_mappings_by_top_id(
@ -791,7 +838,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
current_pod = db_api.get_pod(t_ctx, pod_id)
res = self._get_ports_from_pod_with_number(
context, current_pod, limit, marker,
bottom_top_map, top_bottom_map, filters)
bottom_top_map, top_bottom_map, port_pod_map, filters)
else:
res = self._get_ports_from_top_with_number(
context, limit, marker, top_bottom_map, filters)
@ -802,7 +849,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
if current_pod:
res = self._get_ports_from_pod_with_number(
context, current_pod, limit, '',
bottom_top_map, top_bottom_map, filters)
bottom_top_map, top_bottom_map, port_pod_map, filters)
else:
res = self._get_ports_from_top_with_number(
context, limit, marker, top_bottom_map, filters)
@ -829,7 +876,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
_filters.append(
{'key': key, 'comparator': 'eq',
'value': 'ip_address=%s' % value[
'ip_address']})
'ip_address'][0]})
continue
id_list = self._get_map_filter_ids(
key, value, pod['pod_id'], top_bottom_map)
@ -842,7 +889,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
'comparator': 'eq',
'value': value})
client = self._get_client(pod['region_name'])
ret.extend(client.list_ports(t_ctx, filters=_filters))
ports = client.list_ports(t_ctx, filters=_filters)
ret.extend(self._filter_shadow_port(ports, pod['pod_id'],
port_pod_map))
ret = self._map_ports_from_bottom_to_top(ret, bottom_top_map)
ret.extend(self._get_ports_from_top(context, top_bottom_map,
filters))
@ -854,6 +903,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
def _delete_top_bridge_resource(self, t_ctx, q_ctx, resource_type,
resource_id, resource_name):
# first we update the routing entry to clear bottom_id and expire the
# entry, if we succeed to delete the bridge resource next, we continue
# to delete this expired entry; otherwise, we fail to delete the bridge
# resource, then when the resource is accessed via lock_handle module,
# that module will find the resource and update the entry
with t_ctx.session.begin():
core.update_resources(
t_ctx, models.ResourceRouting,
@ -877,26 +931,20 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
def _delete_top_bridge_network_subnet(self, t_ctx, q_ctx):
project_id = t_ctx.project_id
ew_bridge_subnet_name = t_constants.ew_bridge_subnet_name % project_id
ns_bridge_subnet_name = t_constants.ns_bridge_subnet_name % project_id
subnet_names = [ew_bridge_subnet_name, ns_bridge_subnet_name]
for subnet_name in subnet_names:
bridge_subnet_name = t_constants.bridge_subnet_name % project_id
bridge_subnets = super(TricirclePlugin, self).get_subnets(
q_ctx, {'name': [subnet_name]})
q_ctx, {'name': [bridge_subnet_name]})
if bridge_subnets:
self._delete_top_bridge_resource(
t_ctx, q_ctx, t_constants.RT_SUBNET,
bridge_subnets[0]['id'], subnet_name)
ew_bridge_net_name = t_constants.ew_bridge_net_name % project_id
ns_bridge_net_name = t_constants.ns_bridge_net_name % project_id
net_names = [ew_bridge_net_name, ns_bridge_net_name]
for net_name in net_names:
bridge_subnets[0]['id'], bridge_subnet_name)
bridge_net_name = t_constants.bridge_net_name % project_id
bridge_nets = super(TricirclePlugin, self).get_networks(
q_ctx, {'name': [net_name]})
q_ctx, {'name': [bridge_net_name]})
if bridge_nets:
self._delete_top_bridge_resource(
t_ctx, q_ctx, t_constants.RT_NETWORK, bridge_nets[0]['id'],
net_name)
bridge_net_name)
def _delete_top_bridge_port(self, t_ctx, q_ctx, bridge_port_id,
bridge_port_name):
@ -912,56 +960,37 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
t_constants.RT_ROUTER)
for pod, b_router_id in mappings:
b_client = self._get_client(pod['region_name'])
ew_port_name = t_constants.ew_bridge_port_name % (project_id,
bridge_port_name = t_constants.bridge_port_name % (project_id,
b_router_id)
ew_ports = super(TricirclePlugin, self).get_ports(
context, {'name': [ew_port_name]})
if ew_ports:
t_ew_port_id = ew_ports[0]['id']
b_ew_port_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_ew_port_id, pod['region_name'],
t_constants.RT_PORT)
if b_ew_port_id:
request_body = {'port_id': b_ew_port_id}
try:
b_client.action_routers(t_ctx, 'remove_interface',
b_router_id, request_body)
except Exception as e:
if e.status_code == 404:
# 404 error means that the router interface has
# been already detached, skip this exception
pass
raise
db_api.delete_mappings_by_top_id(t_ctx, t_ew_port_id)
self._delete_top_bridge_port(t_ctx, context, t_ew_port_id,
ew_port_name)
ns_port_name = t_constants.ns_bridge_port_name % (
project_id, b_router_id, None)
ns_ports = super(TricirclePlugin, self).get_ports(
context, {'name': [ns_port_name]})
if ns_ports:
t_ns_port_id = ns_ports[0]['id']
bridge_ports = super(TricirclePlugin, self).get_ports(
context, {'name': [bridge_port_name]})
if bridge_ports:
t_ns_port_id = bridge_ports[0]['id']
b_client.action_routers(t_ctx, 'remove_gateway', b_router_id)
self._delete_top_bridge_port(t_ctx, context, t_ns_port_id,
ns_port_name)
else:
ns_subnet_name = t_constants.ns_bridge_subnet_name % project_id
ns_subnets = super(TricirclePlugin,
bridge_port_name)
b_client.delete_routers(t_ctx, b_router_id)
db_api.delete_mappings_by_bottom_id(t_ctx, b_router_id)
mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, _id, t_constants.RT_NS_ROUTER)
for pod, b_ns_router_id in mappings:
b_client = self._get_client(pod['region_name'])
bridge_subnet_name = t_constants.bridge_subnet_name % project_id
bridge_subnets = super(TricirclePlugin,
self).get_subnets(
context, {'name': [ns_subnet_name]})
if ns_subnets:
t_ns_subnet_id = ns_subnets[0]['id']
b_ns_subnet_id = \
context, {'name': [bridge_subnet_name]})
if bridge_subnets:
t_bridge_subnet_id = bridge_subnets[0]['id']
b_bridge_subnet_id = \
db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_ns_subnet_id, pod['region_name'],
t_ctx, t_bridge_subnet_id, pod['region_name'],
t_constants.RT_SUBNET)
if b_ns_subnet_id:
request_body = {'subnet_id': b_ns_subnet_id}
if b_bridge_subnet_id:
request_body = {'subnet_id': b_bridge_subnet_id}
try:
b_client.action_routers(t_ctx, 'remove_interface',
b_router_id, request_body)
b_ns_router_id, request_body)
except Exception as e:
if e.status_code == 404:
# 404 error means that the router interface has
@ -969,8 +998,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
pass
raise
b_client.delete_routers(t_ctx, b_router_id)
db_api.delete_mappings_by_bottom_id(t_ctx, b_router_id)
b_client.delete_routers(t_ctx, b_ns_router_id)
db_api.delete_mappings_by_bottom_id(t_ctx, b_ns_router_id)
routers = super(TricirclePlugin, self).get_routers(
context, {'tenant_id': [project_id]})
@ -989,13 +1018,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
return self.helper.prepare_bottom_element(
t_ctx, project_id, pod, ele, _type, body)
def _get_bridge_subnet_pool_id(self, t_ctx, q_ctx, project_id, pod, is_ew):
if is_ew:
pool_name = t_constants.ew_bridge_subnet_pool_name
pool_cidr = cfg.CONF.client.ew_bridge_cidr
else:
pool_name = t_constants.ns_bridge_subnet_pool_name
pool_cidr = cfg.CONF.client.ns_bridge_cidr
def _get_bridge_subnet_pool_id(self, t_ctx, q_ctx, project_id, pod):
pool_name = t_constants.bridge_subnet_pool_name
pool_cidr = cfg.CONF.client.bridge_cidr
pool_ele = {'id': pool_name}
body = {'subnetpool': {'tenant_id': project_id,
'name': pool_name,
@ -1012,16 +1037,10 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
return pool_id
def _get_bridge_network_subnet(self, t_ctx, q_ctx, project_id, pod,
pool_id, is_ew):
if is_ew:
net_name = t_constants.ew_bridge_net_name % project_id
pool_id):
net_name = t_constants.bridge_net_name % project_id
net_ele = {'id': net_name}
subnet_name = t_constants.ew_bridge_subnet_name % project_id
subnet_ele = {'id': subnet_name}
else:
net_name = t_constants.ns_bridge_net_name % project_id
net_ele = {'id': net_name}
subnet_name = t_constants.ns_bridge_subnet_name % project_id
subnet_name = t_constants.bridge_subnet_name % project_id
subnet_ele = {'id': subnet_name}
is_admin = q_ctx.is_admin
@ -1063,10 +1082,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
return net, subnet
def _get_bridge_interface(self, t_ctx, q_ctx, project_id, pod,
t_net_id, b_router_id, b_port_id, is_ew):
t_net_id, b_router_id):
port_id = self.helper.get_bridge_interface(t_ctx, q_ctx, project_id,
pod, t_net_id, b_router_id,
b_port_id, is_ew)
pod, t_net_id, b_router_id)
return super(TricirclePlugin, self).get_port(q_ctx, port_id)
def _get_bottom_bridge_elements(self, q_ctx, project_id,
@ -1125,11 +1143,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
# TODO(zhiyuan) decide router is distributed or not from pod table
# currently "distributed" is set to False, should add a metadata field
# to pod table, and decide distributed or not from the metadata later
body = {'router': {'name': router_id,
body = {'router': {'name': t_constants.ns_router_name % router_id,
'distributed': False}}
_, b_router_id = self._prepare_bottom_element(
t_ctx, t_router['tenant_id'], pod, t_router,
t_constants.RT_ROUTER, body)
t_constants.RT_NS_ROUTER, body)
# both router and external network in bottom pod are ready, attach
# external network to router in bottom pod.
@ -1156,13 +1174,12 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
# bridge network.
t_pod = db_api.get_top_pod(t_ctx)
project_id = t_router['tenant_id']
pool_id = self._get_bridge_subnet_pool_id(
t_ctx, context, None, t_pod, False)
pool_id = self._get_bridge_subnet_pool_id(t_ctx, context, None, t_pod)
t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet(
t_ctx, context, project_id, t_pod, pool_id, False)
t_ctx, context, project_id, t_pod, pool_id)
(_, _, b_bridge_subnet_id,
b_bridge_net_id) = self._get_bottom_bridge_elements(
context, project_id, pod, t_bridge_net, False, t_bridge_subnet,
context, project_id, pod, t_bridge_net, True, t_bridge_subnet,
None)
# here we attach the bridge network to the router in bottom pod. to
@ -1181,6 +1198,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
is_attach = _is_bridge_network_attached()
if not is_attach:
# no need to explicitly create the top bridge port, the ip reserved
# for router interface will be used.
b_client.action_routers(t_ctx, 'add_interface', b_router_id,
{'subnet_id': b_bridge_subnet_id})
@ -1199,7 +1218,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
region_name = t_network[az_ext.AZ_HINTS][0]
b_router_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, router_id, region_name, t_constants.RT_ROUTER)
t_ctx, router_id, region_name, t_constants.RT_NS_ROUTER)
b_client = self._get_client(region_name)
b_client.action_routers(t_ctx, 'remove_gateway', b_router_id)
@ -1233,12 +1252,15 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
context, router_id, router)
router_data[l3.EXTERNAL_GW_INFO].update(ret[l3.EXTERNAL_GW_INFO])
self._add_router_gateway(context, router_id, router_data)
return ret
else:
self._remove_router_gateway(context, router_id)
return super(TricirclePlugin, self).update_router(
ret = super(TricirclePlugin, self).update_router(
context, router_id, router)
t_ctx = t_context.get_context_from_neutron_context(context)
self.xjob_handler.configure_extra_routes(t_ctx, router_id)
return ret
def add_router_interface(self, context, router_id, interface_info):
t_ctx = t_context.get_context_from_neutron_context(context)
@ -1251,33 +1273,17 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
t_pod = db_api.get_top_pod(t_ctx)
assert t_pod
# bridge network for E-W networking
# bridge network for E-W and N-S networking
pool_id = self._get_bridge_subnet_pool_id(
t_ctx, context, None, t_pod, True)
t_ctx, context, None, t_pod)
self._get_bridge_network_subnet(
t_ctx, context, project_id, t_pod, pool_id, True)
# bridge network for N-S networking
ext_nets = self.get_networks(context, {external_net.EXTERNAL: [True]})
if not ext_nets:
need_ns_bridge = False
else:
ext_net_region_names = set(
[ext_net[az_ext.AZ_HINTS][0] for ext_net in ext_nets])
need_ns_bridge = False
for b_pod in b_pods:
if b_pod['region_name'] not in ext_net_region_names:
need_ns_bridge = True
break
if need_ns_bridge:
pool_id = self._get_bridge_subnet_pool_id(
t_ctx, context, None, t_pod, False)
self._get_bridge_network_subnet(
t_ctx, context, project_id, t_pod, pool_id, False)
t_ctx, context, project_id, t_pod, pool_id)
return_info = super(TricirclePlugin, self).add_router_interface(
context, router_id, interface_info)
if not b_pods:
LOG.debug('Add router interface: no interfaces found, xjob not'
'triggered')
return return_info
try:
if len(b_pods) == 1:

View File

@ -117,7 +117,7 @@ class NetworkHelper(object):
t_ctx, q_ctx, project_id, pod, ele, _type, body)
def get_bridge_interface(self, t_ctx, q_ctx, project_id, pod,
t_net_id, b_router_id, b_port_id, is_ew):
t_net_id, b_router_id):
"""Get or create top bridge interface
:param t_ctx: tricircle context
@ -126,19 +126,10 @@ class NetworkHelper(object):
:param pod: dict of top pod
:param t_net_id: top bridge network id
:param b_router_id: bottom router id
:param b_port_id: needed when creating bridge interface for south-
north network, id of the internal port bound to floating ip
:param is_ew: create the bridge interface for east-west network or
south-north network
:return: bridge interface id
"""
if is_ew:
port_name = t_constants.ew_bridge_port_name % (project_id,
port_name = t_constants.bridge_port_name % (project_id,
b_router_id)
else:
port_name = t_constants.ns_bridge_port_name % (project_id,
b_router_id,
b_port_id)
port_ele = {'id': port_name}
port_body = {
'port': {
@ -174,7 +165,13 @@ class NetworkHelper(object):
"""
def list_resources(t_ctx_, q_ctx, pod_, ele_, _type_):
client = self._get_client(pod_['region_name'])
if _type_ == t_constants.RT_NETWORK:
if _type_ == t_constants.RT_NS_ROUTER:
_type_ = t_constants.RT_ROUTER
value = t_constants.ns_router_name % ele_['id']
elif _type_ == t_constants.RT_SD_PORT:
_type_ = t_constants.RT_PORT
value = t_constants.shadow_port_name % ele_['id']
elif _type_ == t_constants.RT_NETWORK:
value = utils.get_bottom_network_name(ele_)
else:
value = ele_['id']
@ -183,6 +180,10 @@ class NetworkHelper(object):
'value': value}])
def create_resources(t_ctx_, q_ctx, pod_, body_, _type_):
if _type_ == t_constants.RT_NS_ROUTER:
_type_ = t_constants.RT_ROUTER
elif _type_ == t_constants.RT_SD_PORT:
_type_ = t_constants.RT_PORT
client = self._get_client(pod_['region_name'])
return client.create_resources(_type_, t_ctx_, body_)
@ -421,14 +422,6 @@ class NetworkHelper(object):
'cidr': t_subnet['cidr'],
'enable_dhcp': False,
'tenant_id': project_id}}
# In the pod hosting external network, where ns bridge network is used
# as an internal network, need to allocate ip address from .3 because
# .2 is used by the router gateway port in the pod hosting servers,
# where ns bridge network is used as an external network.
# if t_subnet['name'].startswith('ns_bridge_') and not is_external:
# prefix = t_subnet['cidr'][:t_subnet['cidr'].rindex('.')]
# subnet_body['subnet']['allocation_pools'] = [
# {'start': prefix + '.3', 'end': prefix + '.254'}]
_, b_subnet_id = self.prepare_bottom_element(
t_ctx, project_id, pod, t_subnet, 'subnet', subnet_body)

View File

@ -0,0 +1,60 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import orm
from neutron_lib import constants
from neutron.db.models import l3 as l3_models
from neutron.db import models_v2
from neutron.extensions import l3
from neutron.services.l3_router import l3_router_plugin
class TricircleL3Plugin(l3_router_plugin.L3RouterPlugin):
# Override the original implementation to allow associating a floating ip
# to a port whose network is not attached to the router. Tricircle will
# configures extra routes to guarantee packets can reach the port.
def get_router_for_floatingip(self, context, internal_port,
internal_subnet, external_network_id):
"""Find a router to handle the floating-ip association.
:param internal_port: The port for the fixed-ip.
:param internal_subnet: The subnet for the fixed-ip.
:param external_network_id: The external network for floating-ip.
:raises: ExternalGatewayForFloatingIPNotFound if no suitable router
is found.
"""
router_port = l3_models.RouterPort
gw_port = orm.aliased(models_v2.Port, name="gw_port")
router_port_qry = context.session.query(
router_port.router_id
).join(gw_port, gw_port.device_id == router_port.router_id).filter(
gw_port.network_id == external_network_id,
gw_port.device_owner == constants.DEVICE_OWNER_ROUTER_GW
).distinct()
first_router_id = None
for router in router_port_qry:
if not first_router_id:
first_router_id = router.router_id
if first_router_id:
return first_router_id
raise l3.ExternalGatewayForFloatingIPNotFound(
subnet_id=internal_subnet['id'],
external_network_id=external_network_id,
port_id=internal_port['id'])

View File

@ -480,10 +480,15 @@ class TricirclePlugin(plugin.Ml2Plugin):
t_ctx = t_context.get_context_from_neutron_context(context)
try:
b_port = self.core_plugin.get_port(context, _id)
# to support floating ip, we create a copy port if the target port
# is not in the pod where the real external network is located. to
# distinguish it from normal port, we name it with a prefix
do_top_delete = b_port['device_owner'].startswith(
q_constants.DEVICE_OWNER_COMPUTE_PREFIX)
skip_top_delete = t_constants.RT_SD_PORT in b_port['name']
except q_exceptions.NotFound:
return
if b_port['device_owner'].startswith(
q_constants.DEVICE_OWNER_COMPUTE_PREFIX):
if do_top_delete and not skip_top_delete:
self.neutron_handle.handle_delete(t_ctx, t_constants.RT_PORT, _id)
self.core_plugin.delete_port(context, _id, l3_port_check)

View File

@ -294,7 +294,7 @@ class FakeClient(object):
'security_group': BOTTOM2_SGS,
'floatingip': BOTTOM2_FIPS}}
def __init__(self, region_name):
def __init__(self, region_name=None):
if not region_name:
self.region_name = 'top'
else:
@ -336,6 +336,12 @@ class FakeClient(object):
if ip in ip_range:
fixed_ip['subnet_id'] = subnet['id']
break
if 'subnet_id' not in fixed_ip:
# we still cannot find the proper subnet, that's because
# this is a copy port. local plugin will create the missing
# subnet for this port but FakeClient won't. we just skip
# the ip address check
continue
if fixed_ip['ip_address'] in subnet_ips_map.get(
fixed_ip['subnet_id'], set()):
raise q_exceptions.IpAddressInUseClient()
@ -1118,6 +1124,11 @@ class PluginTest(unittest.TestCase,
DotDict({'physical_network': phynet,
'vlan_id': vlan, 'allocated': False}))
def fake_get_plugin(alias=q_constants.CORE):
return FakePlugin()
from neutron_lib.plugins import directory
directory.get_plugin = fake_get_plugin
def _basic_pod_route_setup(self):
pod1 = {'pod_id': 'pod_id_1',
'region_name': 'pod_1',
@ -1440,12 +1451,11 @@ class PluginTest(unittest.TestCase,
# test _prepare_top_element
pool_id = fake_plugin._get_bridge_subnet_pool_id(
t_ctx, q_ctx, 'project_id', t_pod, True)
t_ctx, q_ctx, 'project_id', t_pod)
net, subnet = fake_plugin._get_bridge_network_subnet(
t_ctx, q_ctx, 'project_id', t_pod, pool_id, True)
t_ctx, q_ctx, 'project_id', t_pod, pool_id)
port = fake_plugin._get_bridge_interface(t_ctx, q_ctx, 'project_id',
pod, net['id'], 'b_router_id',
None, True)
pod, net['id'], 'b_router_id')
top_entry_map = {}
with t_ctx.session.begin():
@ -1485,10 +1495,8 @@ class PluginTest(unittest.TestCase,
@staticmethod
def _prepare_network_test(tenant_id, ctx, region_name, index):
t_net_id = uuidutils.generate_uuid()
t_subnet_id = uuidutils.generate_uuid()
b_net_id = uuidutils.generate_uuid()
b_subnet_id = uuidutils.generate_uuid()
t_net_id = b_net_id = uuidutils.generate_uuid()
t_subnet_id = b_subnet_id = uuidutils.generate_uuid()
# no need to specify az, we will setup router in the pod where bottom
# network is created
@ -1632,9 +1640,10 @@ class PluginTest(unittest.TestCase,
'_allocate_ips_for_port', new=fake_allocate_ips_for_port)
@patch.object(db_base_plugin_common.DbBasePluginCommon,
'_make_subnet_dict', new=fake_make_subnet_dict)
@patch.object(FakeClient, 'add_gateway_routers')
@patch.object(FakeBaseRPCAPI, 'configure_extra_routes')
@patch.object(context, 'get_context_from_neutron_context')
def test_add_interface(self, mock_context, mock_rpc):
def test_add_interface(self, mock_context, mock_rpc, mock_action):
self._basic_pod_route_setup()
fake_plugin = FakePlugin()
@ -1651,185 +1660,63 @@ class PluginTest(unittest.TestCase,
q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id']
_, b_router_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_router_id, 'router')[0]
t_ctx, t_router_id, constants.RT_ROUTER)[0]
mock_rpc.assert_called_once_with(t_ctx, t_router_id)
for b_net in BOTTOM1_NETS:
if 'provider:segmentation_id' in b_net:
self.assertIn(b_net['provider:segmentation_id'], (2000, 2001))
# only one VLAN allocated, for E-W bridge network
# only one VLAN allocated since we just create one bridge network
allocations = [
allocation['allocated'] for allocation in TOP_VLANALLOCATIONS]
self.assertItemsEqual([True, False], allocations)
for segment in TOP_SEGMENTS:
self.assertIn(segment['segmentation_id'], (2000, 2001))
bridge_port_name = constants.ew_bridge_port_name % (tenant_id,
bridge_port_name = constants.bridge_port_name % (tenant_id,
b_router_id)
_, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, bridge_port_name, 'port')[0]
_, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_bridge_port_id, 'port')[0]
(t_net_id, t_subnet_id, t_router_id,
b_another_net_id, b_another_subnet_id) = self._prepare_router_test(
tenant_id, t_ctx, 'pod_1', 2)
fake_plugin.add_router_interface(
q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id']
t_ns_bridge_net_id = None
for net in TOP_NETS:
if net['name'].startswith('ns_bridge'):
t_ns_bridge_net_id = net['id']
# N-S bridge not created since no external network created
self.assertIsNone(t_ns_bridge_net_id)
device_ids = ['', '', '']
for port in BOTTOM1_PORTS:
if port['id'] == b_bridge_port_id:
device_ids[0] = port['device_id']
elif port['network_id'] == b_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[1] = port['device_id']
elif port['network_id'] == b_another_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[2] = port['device_id']
self.assertEqual(device_ids, [b_router_id, b_router_id, b_router_id])
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@patch.object(ipam_pluggable_backend.IpamPluggableBackend,
'_allocate_ips_for_port', new=fake_allocate_ips_for_port)
@patch.object(db_base_plugin_common.DbBasePluginCommon,
'_make_subnet_dict', new=fake_make_subnet_dict)
@patch.object(FakeBaseRPCAPI, 'configure_extra_routes')
@patch.object(FakeClient, 'add_gateway_routers')
@patch.object(context, 'get_context_from_neutron_context')
def test_add_interface_with_external_network(self, mock_context,
mock_action, mock_rpc):
self._basic_pod_route_setup()
fake_plugin = FakePlugin()
q_ctx = FakeNeutronContext()
t_ctx = context.get_db_context()
mock_context.return_value = t_ctx
tenant_id = TEST_TENANT_ID
(t_net_id, t_subnet_id,
t_router_id, b_net_id, b_subnet_id) = self._prepare_router_test(
tenant_id, t_ctx, 'pod_1', 1)
e_net_id = uuidutils.generate_uuid()
e_net = {'id': e_net_id,
'name': 'ext-net',
'admin_state_up': True,
'shared': False,
'tenant_id': tenant_id,
'router:external': True,
'availability_zone_hints': '["pod_2"]'}
TOP_NETS.append(e_net)
fake_plugin.add_router_interface(
q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id']
b_router_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_router_id, 'pod_1', 'router')
mock_rpc.assert_called_once_with(t_ctx, t_router_id)
for b_net in BOTTOM1_NETS:
if 'provider:segmentation_id' in b_net:
self.assertIn(b_net['provider:segmentation_id'], (2000, 2001))
# two VLANs allocated, for E-W and N-S bridge network
allocations = [
allocation['allocated'] for allocation in TOP_VLANALLOCATIONS]
self.assertItemsEqual([True, True], allocations)
for segment in TOP_SEGMENTS:
self.assertIn(segment['segmentation_id'], (2000, 2001))
bridge_port_name = constants.ew_bridge_port_name % (tenant_id,
b_router_id)
_, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, bridge_port_name, 'port')[0]
_, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_bridge_port_id, 'port')[0]
(t_net_id, t_subnet_id, t_router_id,
b_another_net_id, b_another_subnet_id) = self._prepare_router_test(
tenant_id, t_ctx, 'pod_1', 2)
fake_plugin.add_router_interface(
q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id']
for net in TOP_NETS:
if net['name'].startswith('ns_bridge'):
t_ns_bridge_net_id = net['id']
for subnet in TOP_SUBNETS:
if subnet['name'].startswith('ns_bridge'):
t_ns_bridge_subnet_id = subnet['id']
for t_port in TOP_PORTS:
if t_port['id'] == t_bridge_port_id:
t_ns_bridge_net_id = t_port['network_id']
t_ns_bridge_subnet_id = t_port['fixed_ips'][0]['subnet_id']
b_ns_bridge_net_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_ns_bridge_net_id, 'pod_1', constants.RT_NETWORK)
b_ns_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_ns_bridge_subnet_id, 'pod_1', constants.RT_SUBNET)
# internal network and external network are in different pods, need
# to create N-S bridge network and set gateway, add_router_interface
# is called two times, so add_gateway is also called two times.
# add_interface is called three times because the second time
# add_router_interface is called, bottom router is already attached
# to E-W bridge network, only need to attach internal network to
# bottom router
calls = [mock.call(t_ctx, b_router_id,
{'network_id': b_ns_bridge_net_id,
'external_fixed_ips': [
{'subnet_id': b_ns_bridge_subnet_id,
'ip_address': '100.128.0.2'}]}),
mock.call(t_ctx, b_router_id,
{'network_id': b_ns_bridge_net_id,
'external_fixed_ips': [
{'subnet_id': b_ns_bridge_subnet_id,
'ip_address': '100.128.0.2'}]})]
mock_action.assert_has_calls(calls)
device_ids = ['', '', '']
for port in BOTTOM1_PORTS:
if port['id'] == b_bridge_port_id:
device_ids[0] = port['device_id']
elif port['network_id'] == b_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[1] = port['device_id']
elif port['network_id'] == b_another_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[2] = port['device_id']
self.assertEqual(device_ids, [b_router_id, b_router_id, b_router_id])
(t_net_id, t_subnet_id,
t_router_id, b_net_id, b_subnet_id) = self._prepare_router_test(
tenant_id, t_ctx, 'pod_2', 2)
(t_net_id, t_subnet_id, t_router_id,
b_another_net_id, b_another_subnet_id) = self._prepare_router_test(
tenant_id, t_ctx, 'pod_1', 2)
fake_plugin.add_router_interface(
q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id']
b_router_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_router_id, 'pod_2', 'router')
bridge_port_name = constants.ew_bridge_port_name % (tenant_id,
b_router_id)
_, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, bridge_port_name, 'port')[0]
_, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_bridge_port_id, 'port')[0]
# internal network and external network are in the same pod, no need
# to create N-S bridge network when attaching router interface(N-S
# bridge network is created when setting router external gateway), so
# add_gateway is not called.
mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_router_id, constants.RT_NS_ROUTER)
# router for north-south networking is not created since no external
# network created
self.assertEqual(len(mappings), 0)
device_ids = ['', '']
for port in BOTTOM2_PORTS:
if port['id'] == b_bridge_port_id:
for port in BOTTOM1_PORTS:
if port['network_id'] == b_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[0] = port['device_id']
elif port['network_id'] == b_net_id and (
elif port['network_id'] == b_another_net_id and (
port['device_owner'] == 'network:router_interface'):
device_ids[1] = port['device_id']
self.assertEqual(device_ids, [b_router_id, b_router_id])
call = mock.call(t_ctx, b_router_id,
{'network_id': b_ns_bridge_net_id,
'enable_snat': False,
'external_fixed_ips': [
{'subnet_id': b_ns_bridge_subnet_id,
'ip_address': '100.0.0.2'}]})
# each router interface adding will call add_gateway once
mock_action.assert_has_calls([call, call])
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@ -1870,24 +1757,25 @@ class PluginTest(unittest.TestCase,
[{'key': 'resource_type',
'comparator': 'eq',
'value': 'port'}], [])
# two new entries, for top and bottom bridge ports
self.assertEqual(entry_num + 2, len(entries))
# top and bottom interface is deleted, only bridge port left
# one new entry, for top bridge port
self.assertEqual(entry_num + 1, len(entries))
# top and bottom interface is deleted, only top bridge port left
self.assertEqual(1, len(TOP_PORTS))
self.assertEqual(1, len(BOTTOM1_PORTS))
self.assertEqual(0, len(BOTTOM1_PORTS))
mock_action.side_effect = None
fake_plugin.add_router_interface(q_ctx, t_router_id,
{'subnet_id': t_subnet_id})
# bottom dhcp port and bridge port
self.assertEqual(2, len(BOTTOM1_PORTS))
# just bottom dhcp port, bottom interface is not created because
# action_routers function is mocked
self.assertEqual(1, len(BOTTOM1_PORTS))
with t_ctx.session.begin():
entries = core.query_resource(t_ctx, models.ResourceRouting,
[{'key': 'resource_type',
'comparator': 'eq',
'value': 'port'}], [])
# three more entries, for top and bottom dhcp ports, top interface
self.assertEqual(entry_num + 2 + 3, len(entries))
self.assertEqual(entry_num + 1 + 3, len(entries))
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@ -1921,8 +1809,8 @@ class PluginTest(unittest.TestCase,
# test that we can success when bottom pod comes back
fake_plugin.add_router_interface(
q_ctx, t_router_id, {'subnet_id': t_subnet_id})
# bottom dhcp port, bottom interface and bridge port
self.assertEqual(3, len(BOTTOM1_PORTS))
# bottom dhcp port and bottom interface
self.assertEqual(2, len(BOTTOM1_PORTS))
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@ -2082,24 +1970,24 @@ class PluginTest(unittest.TestCase,
'external_fixed_ips': [{'subnet_id': TOP_SUBNETS[0]['id'],
'ip_address': '100.64.0.5'}]}}})
b_router_id = BOTTOM1_ROUTERS[0]['id']
b_ns_router_id = BOTTOM1_ROUTERS[0]['id']
b_net_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_net_id, 'pod_1', constants.RT_NETWORK)
b_subnet_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_subnet_id, 'pod_1', constants.RT_SUBNET)
for subnet in TOP_SUBNETS:
if subnet['name'].startswith('ns_bridge_subnet'):
t_ns_bridge_subnet_id = subnet['id']
b_ns_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_ns_bridge_subnet_id, 'pod_1', constants.RT_SUBNET)
if subnet['name'].startswith('bridge_subnet'):
t_bridge_subnet_id = subnet['id']
b_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, t_bridge_subnet_id, 'pod_1', constants.RT_SUBNET)
body = {'network_id': b_net_id,
'enable_snat': False,
'external_fixed_ips': [{'subnet_id': b_subnet_id,
'ip_address': '100.64.0.5'}]}
calls = [mock.call(t_ctx, 'add_gateway', b_router_id, body),
mock.call(t_ctx, 'add_interface', b_router_id,
{'subnet_id': b_ns_bridge_subnet_id})]
calls = [mock.call(t_ctx, 'add_gateway', b_ns_router_id, body),
mock.call(t_ctx, 'add_interface', b_ns_router_id,
{'subnet_id': b_bridge_subnet_id})]
mock_action.assert_has_calls(calls)
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@ -2165,14 +2053,14 @@ class PluginTest(unittest.TestCase,
'enable_snat': False,
'external_fixed_ips': [{'subnet_id': t_subnet_id,
'ip_address': '100.64.0.5'}]}}})
_, b_router_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_router_id, constants.RT_ROUTER)[0]
_, b_ns_router_id = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_router_id, constants.RT_NS_ROUTER)[0]
# then remove router gateway
fake_plugin.update_router(
q_ctx, t_router_id,
{'router': {'external_gateway_info': {}}})
mock_action.assert_called_with(t_ctx, 'remove_gateway', b_router_id)
mock_action.assert_called_with(t_ctx, 'remove_gateway', b_ns_router_id)
def _prepare_associate_floatingip_test(self, t_ctx, q_ctx, fake_plugin):
tenant_id = TEST_TENANT_ID
@ -2221,7 +2109,8 @@ class PluginTest(unittest.TestCase,
{'subnet_id': t_subnet_id})
# create internal port
t_port_id = uuidutils.generate_uuid()
b_port_id = uuidutils.generate_uuid()
# now top id and bottom id are the same
b_port_id = t_port_id
t_port = {
'id': t_port_id,
'network_id': t_net_id,
@ -2272,6 +2161,12 @@ class PluginTest(unittest.TestCase,
(t_port_id, b_port_id,
fip, e_net) = self._prepare_associate_floatingip_test(t_ctx, q_ctx,
fake_plugin)
# here we attach a subnet in pod2 to the router to test if the two
# bottom routers can be successfully created
_, t_subnet_id2, t_router_id, _, _ = self._prepare_router_test(
TEST_TENANT_ID, t_ctx, 'pod_2', 2)
fake_plugin.add_router_interface(q_ctx, t_router_id,
{'subnet_id': t_subnet_id2})
# associate floating ip
fip_body = {'port_id': t_port_id}
@ -2280,25 +2175,28 @@ class PluginTest(unittest.TestCase,
b_ext_net_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, e_net['id'], 'pod_2', constants.RT_NETWORK)
for port in BOTTOM2_PORTS:
if port['name'] == 'ns_bridge_port':
ns_bridge_port = port
for net in TOP_NETS:
if net['name'].startswith('ns_bridge'):
b_bridge_net_id = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, net['id'], 'pod_1', constants.RT_NETWORK)
calls = [mock.call(t_ctx,
{'floatingip': {
'floating_network_id': b_bridge_net_id,
'floating_ip_address': '100.128.0.3',
'port_id': b_port_id}}),
mock.call(t_ctx,
{'floatingip': {
'floating_network_id': b_ext_net_id,
'floating_ip_address': fip[
'floating_ip_address'],
'port_id': ns_bridge_port['id']}})]
'port_id': b_port_id}})]
mock_create.assert_has_calls(calls)
# routers for east-west networking and north-south networking
self.assertEqual(2, len(BOTTOM2_ROUTERS))
# check routing entries for copied resources have been created
fake_client = FakeClient()
t_port = fake_client.get_ports(t_ctx, t_port_id)
cp_port_mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_port_id, constants.RT_SD_PORT)
cp_subnet_mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_port['fixed_ips'][0]['subnet_id'], constants.RT_SD_SUBNET)
cp_network_mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_port['network_id'], constants.RT_SD_NETWORK)
self.assertEqual(1, len(cp_port_mappings))
self.assertEqual(1, len(cp_subnet_mappings))
self.assertEqual(1, len(cp_network_mappings))
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@ -2364,34 +2262,22 @@ class PluginTest(unittest.TestCase,
fake_plugin.update_floatingip(q_ctx, fip['id'],
{'floatingip': fip_body})
bridge_port_name = constants.ns_bridge_port_name % (
e_net['tenant_id'], None, b_port_id)
t_pod = db_api.get_top_pod(t_ctx)
mapping = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT)
# check routing for bridge port in top pod exists
self.assertIsNotNone(mapping)
# disassociate floating ip
fip_body = {'port_id': None}
fake_plugin.update_floatingip(q_ctx, fip['id'],
{'floatingip': fip_body})
fip_id1 = BOTTOM1_FIPS[0]['id']
fip_id2 = BOTTOM2_FIPS[0]['id']
calls = [mock.call(t_ctx, fip_id1),
mock.call(t_ctx, fip_id2)]
mock_delete.assert_has_calls(calls)
mapping = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT)
# check routing for bridge port in top pod is deleted
self.assertIsNone(mapping)
fip_id = BOTTOM2_FIPS[0]['id']
mock_delete.assert_called_once_with(t_ctx, fip_id)
# check the association information is cleared
self.assertIsNone(TOP_FLOATINGIPS[0]['fixed_port_id'])
self.assertIsNone(TOP_FLOATINGIPS[0]['fixed_ip_address'])
self.assertIsNone(TOP_FLOATINGIPS[0]['router_id'])
# check routing entry for copied port has been created
cp_port_mappings = db_api.get_bottom_mappings_by_top_id(
t_ctx, t_port_id, constants.RT_SD_PORT)
self.assertEqual(0, len(cp_port_mappings))
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
@ -2419,26 +2305,11 @@ class PluginTest(unittest.TestCase,
fake_plugin.update_floatingip(q_ctx, fip['id'],
{'floatingip': fip_body})
bridge_port_name = constants.ns_bridge_port_name % (
e_net['tenant_id'], None, b_port_id)
t_pod = db_api.get_top_pod(t_ctx)
mapping = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT)
# check routing for bridge port in top pod exists
self.assertIsNotNone(mapping)
# disassociate floating ip
fake_plugin.delete_floatingip(q_ctx, fip['id'])
fip_id1 = BOTTOM1_FIPS[0]['id']
fip_id2 = BOTTOM2_FIPS[0]['id']
calls = [mock.call(t_ctx, fip_id1),
mock.call(t_ctx, fip_id2)]
mock_delete.assert_has_calls(calls)
mapping = db_api.get_bottom_id_by_top_id_region_name(
t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT)
# check routing for bridge port in top pod is deleted
self.assertIsNone(mapping)
fip_id = BOTTOM2_FIPS[0]['id']
mock_delete.assert_called_once_with(t_ctx, fip_id)
@patch.object(directory, 'get_plugin', new=fake_get_plugin)
@patch.object(driver.Pool, 'get_instance', new=fake_get_instance)

View File

@ -38,27 +38,36 @@ BOTTOM1_SUBNET = []
BOTTOM2_SUBNET = []
BOTTOM1_PORT = []
BOTTOM2_PORT = []
TOP_ROUTER = []
BOTTOM1_ROUTER = []
BOTTOM2_ROUTER = []
TOP_SG = []
BOTTOM1_SG = []
BOTTOM2_SG = []
TOP_FIP = []
BOTTOM1_FIP = []
BOTTOM2_FIP = []
RES_LIST = [TOP_NETWORK, BOTTOM1_NETWORK, BOTTOM2_NETWORK, TOP_SUBNET,
BOTTOM1_SUBNET, BOTTOM2_SUBNET, BOTTOM1_PORT, BOTTOM2_PORT,
BOTTOM1_ROUTER, BOTTOM2_ROUTER, TOP_SG, BOTTOM1_SG, BOTTOM2_SG]
TOP_ROUTER, BOTTOM1_ROUTER, BOTTOM2_ROUTER, TOP_SG, BOTTOM1_SG,
BOTTOM2_SG, TOP_FIP, BOTTOM1_FIP, BOTTOM2_FIP]
RES_MAP = {'top': {'network': TOP_NETWORK,
'subnet': TOP_SUBNET,
'security_group': TOP_SG},
'router': TOP_ROUTER,
'security_group': TOP_SG,
'floatingips': TOP_FIP},
'pod_1': {'network': BOTTOM1_NETWORK,
'subnet': BOTTOM1_SUBNET,
'port': BOTTOM1_PORT,
'router': BOTTOM1_ROUTER,
'security_group': BOTTOM1_SG},
'security_group': BOTTOM1_SG,
'floatingips': BOTTOM1_FIP},
'pod_2': {'network': BOTTOM2_NETWORK,
'subnet': BOTTOM2_SUBNET,
'port': BOTTOM2_PORT,
'router': BOTTOM2_ROUTER,
'security_group': BOTTOM2_SG}}
'security_group': BOTTOM2_SG,
'floatingips': BOTTOM2_FIP}}
class FakeXManager(xmanager.XManager):
@ -84,7 +93,7 @@ class FakeClient(object):
if _filter['key'] not in res:
is_selected = False
break
if res[_filter['key']] != _filter['value']:
if res[_filter['key']] not in _filter['value']:
is_selected = False
break
if is_selected:
@ -102,6 +111,11 @@ class FakeClient(object):
'subnet', cxt,
[{'key': 'id', 'comparator': 'eq', 'value': subnet_id}])[0]
def get_routers(self, cxt, router_id):
return self.list_resources(
'router', cxt,
[{'key': 'id', 'comparator': 'eq', 'value': router_id}])[0]
def update_routers(self, cxt, *args, **kwargs):
pass
@ -119,6 +133,9 @@ class FakeClient(object):
def create_security_group_rules(self, cxt, *args, **kwargs):
pass
def list_floatingips(self, cxt, filters=None):
return self.list_resources('floatingips', cxt, filters)
class XManagerTest(unittest.TestCase):
def setUp(self):
@ -132,11 +149,68 @@ class XManagerTest(unittest.TestCase):
cfg.CONF.register_opt(opt)
self.context = context.Context()
self.xmanager = FakeXManager()
self.xmanager = FakeXManager()
@patch.object(FakeClient, 'update_routers')
def test_configure_extra_routes(self, mock_update):
top_router_id = 'router_id'
def _prepare_dnat_test(self):
for subnet in BOTTOM2_SUBNET:
if 'ext' in subnet['id']:
ext_subnet = subnet
ext_cidr = ext_subnet['cidr']
ext_cidr_prefix = ext_cidr[:ext_cidr.rindex('.')]
vm_ports = []
# get one vm port from each bottom pod
for ports in [BOTTOM1_PORT, BOTTOM2_PORT]:
for port in ports:
if port['device_owner'] == 'compute:None':
vm_ports.append(port)
break
for i, vm_port in enumerate(vm_ports):
vm_ip = vm_port['fixed_ips'][0]['ip_address']
fip = {'floating_network_id': ext_subnet['network_id'],
'floating_ip_address': '%s.%d' % (ext_cidr_prefix, i + 1),
'port_id': vm_port['id'],
'fixed_ip_address': vm_ip}
TOP_FIP.append(fip)
BOTTOM2_FIP.append(fip)
def _prepare_snat_test(self, top_router_id):
ext_network = {'id': 'ext_network_id',
'router:external': True}
ext_subnet = {
'id': 'ext_subnet_id',
'network_id': ext_network['id'],
'cidr': '162.3.124.0/24',
'gateway_ip': '162.3.124.1'
}
for router in TOP_ROUTER:
if router['id'] == top_router_id:
router['external_gateway_info'] = {
'network_id': ext_network['id']}
router = {'id': 'ns_router_id'}
for subnet in BOTTOM2_SUBNET:
if 'bridge' in subnet['id']:
bridge_subnet = subnet
bridge_port = {
'network_id': bridge_subnet['network_id'],
'device_id': router['id'],
'device_owner': 'network:router_interface',
'fixed_ips': [{'subnet_id': bridge_subnet['id'],
'ip_address': bridge_subnet['gateway_ip']}]
}
BOTTOM2_NETWORK.append(ext_network)
BOTTOM2_SUBNET.append(ext_subnet)
BOTTOM2_PORT.append(bridge_port)
BOTTOM2_ROUTER.append(router)
route = {'top_id': top_router_id, 'bottom_id': router['id'],
'pod_id': 'pod_id_2', 'resource_type': constants.RT_NS_ROUTER}
with self.context.session.begin():
core.create_resource(self.context, models.ResourceRouting, route)
return bridge_subnet['gateway_ip'], router['id']
def _prepare_east_west_network_test(self, top_router_id):
bridge_infos = []
router = {'id': top_router_id}
TOP_ROUTER.append(router)
for i in xrange(1, 3):
pod_dict = {'pod_id': 'pod_id_%d' % i,
'region_name': 'pod_%d' % i,
@ -156,7 +230,7 @@ class XManagerTest(unittest.TestCase):
'id': 'bridge_subnet_%d_id' % i,
'network_id': bridge_network['id'],
'cidr': '100.0.1.0/24',
'gateway_ip': '100.0.1.%d' % i,
'gateway_ip': '100.0.1.1',
}
port = {
'network_id': network['id'],
@ -166,18 +240,22 @@ class XManagerTest(unittest.TestCase):
'ip_address': subnet['gateway_ip']}]
}
vm_port = {
'id': 'vm_port_%d_id' % i,
'network_id': network['id'],
'device_id': 'vm%d_id' % i,
'device_owner': 'compute:None',
'fixed_ips': [{'subnet_id': subnet['id'],
'ip_address': '10.0.%d.3' % i}]
}
bridge_cidr = bridge_subnet['cidr']
bridge_port_ip = '%s.%d' % (bridge_cidr[:bridge_cidr.rindex('.')],
2 + i)
bridge_port = {
'network_id': bridge_network['id'],
'device_id': router['id'],
'device_owner': 'network:router_interface',
'device_owner': 'network:router_gateway',
'fixed_ips': [{'subnet_id': bridge_subnet['id'],
'ip_address': bridge_subnet['gateway_ip']}]
'ip_address': bridge_port_ip}]
}
region_name = 'pod_%d' % i
RES_MAP[region_name]['network'].append(network)
@ -194,6 +272,13 @@ class XManagerTest(unittest.TestCase):
with self.context.session.begin():
core.create_resource(self.context, models.ResourceRouting,
route)
bridge_info = {
'router_id': router['id'],
'bridge_ip': bridge_port['fixed_ips'][0]['ip_address'],
'vm_ips': ['10.0.%d.3' % i]}
bridge_infos.append(bridge_info)
BOTTOM1_NETWORK.append({'id': 'network_3_id'})
BOTTOM1_SUBNET.append({'id': 'subnet_3_id',
'network_id': 'network_3_id',
@ -209,30 +294,88 @@ class XManagerTest(unittest.TestCase):
'device_owner': 'compute:None',
'fixed_ips': [{'subnet_id': 'subnet_3_id',
'ip_address': '10.0.3.3'}]})
bridge_infos[0]['vm_ips'].append('10.0.3.3')
return bridge_infos
def _check_extra_routes_calls(self, except_list, actual_list):
except_map = {}
for except_call in except_list:
ctx, router_id, routes_body = except_call[1]
except_map[router_id] = (ctx, routes_body['router']['routes'])
for actual_call in actual_list:
ctx, router_id, routes_body = actual_call[0]
expect_ctx, expect_routes = except_map[router_id]
self.assertEqual(expect_ctx, ctx)
self.assertItemsEqual(expect_routes,
routes_body['router']['routes'])
@patch.object(FakeClient, 'update_routers')
def test_configure_extra_routes_with_floating_ips(self, mock_update):
top_router_id = 'router_id'
bridge_infos = self._prepare_east_west_network_test(top_router_id)
ns_bridge_ip, ns_router_id = self._prepare_snat_test(top_router_id)
self._prepare_dnat_test()
self.xmanager.configure_extra_routes(self.context,
payload={'router': top_router_id})
calls = [mock.call(self.context, 'router_1_id',
{'router': {
'routes': [{'nexthop': '100.0.1.2',
'destination': '10.0.2.3/32'}]}}),
mock.call(self.context, 'router_2_id',
{'router': {
'routes': [{'nexthop': '100.0.1.1',
'destination': '10.0.1.3/32'},
{'nexthop': '100.0.1.1',
'destination': '10.0.3.3/32'}]}}),
mock.call(self.context, 'router_2_id',
{'router': {
'routes': [{'nexthop': '100.0.1.1',
'destination': '10.0.3.3/32'},
{'nexthop': '100.0.1.1',
'destination': '10.0.1.3/32'}]}})]
calls = []
ns_routes = []
for i in range(2):
routes = []
for ip in bridge_infos[i]['vm_ips']:
route = {'nexthop': bridge_infos[i]['bridge_ip'],
'destination': ip + '/32'}
routes.append(route)
ns_routes.append(route)
routes.append({'nexthop': ns_bridge_ip,
'destination': '0.0.0.0/0'})
call = mock.call(self.context, bridge_infos[1 - i]['router_id'],
{'router': {'routes': routes}})
calls.append(call)
calls.append(mock.call(self.context, ns_router_id,
{'router': {'routes': ns_routes}}))
self._check_extra_routes_calls(calls, mock_update.call_args_list)
called = mock_update.call_args_list[1] == calls[1]
called = called or (mock_update.call_args_list[1] == calls[2])
called = called and (mock_update.call_args_list[0] == calls[0])
self.assertTrue(called)
@patch.object(FakeClient, 'update_routers')
def test_configure_extra_routes_with_external_network(self, mock_update):
top_router_id = 'router_id'
bridge_infos = self._prepare_east_west_network_test(top_router_id)
ns_bridge_ip, ns_router_id = self._prepare_snat_test(top_router_id)
self.xmanager.configure_extra_routes(self.context,
payload={'router': top_router_id})
calls = []
ns_routes = []
for i in range(2):
routes = []
for ip in bridge_infos[i]['vm_ips']:
route = {'nexthop': bridge_infos[i]['bridge_ip'],
'destination': ip + '/32'}
routes.append(route)
ns_routes.append(route)
routes.append({'nexthop': ns_bridge_ip,
'destination': '0.0.0.0/0'})
call = mock.call(self.context, bridge_infos[1 - i]['router_id'],
{'router': {'routes': routes}})
calls.append(call)
calls.append(mock.call(self.context, ns_router_id,
{'router': {'routes': ns_routes}}))
self._check_extra_routes_calls(calls, mock_update.call_args_list)
@patch.object(FakeClient, 'update_routers')
def test_configure_extra_routes(self, mock_update):
top_router_id = 'router_id'
bridge_infos = self._prepare_east_west_network_test(top_router_id)
self.xmanager.configure_extra_routes(self.context,
payload={'router': top_router_id})
calls = []
for i in range(2):
routes = []
for ip in bridge_infos[i]['vm_ips']:
routes.append({'nexthop': bridge_infos[i]['bridge_ip'],
'destination': ip + '/32'})
call = mock.call(self.context, bridge_infos[1 - i]['router_id'],
{'router': {'routes': routes}})
calls.append(call)
self._check_extra_routes_calls(calls, mock_update.call_args_list)
@patch.object(FakeClient, 'delete_security_group_rules')
@patch.object(FakeClient, 'create_security_group_rules')

View File

@ -24,6 +24,7 @@ from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_service import periodic_task
import neutron_lib.constants as q_constants
import neutron_lib.exceptions as q_exceptions
import neutronclient.common.exceptions as q_cli_exceptions
@ -37,7 +38,6 @@ from tricircle.db import models
import tricircle.network.exceptions as t_network_exc
from tricircle.network import helper
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -294,8 +294,13 @@ class XManager(PeriodicTasks):
raise
def _setup_router_one_pod(self, ctx, t_pod, b_pod, t_client, t_net,
t_router, t_ew_bridge_net, t_ew_bridge_subnet,
need_ns_bridge):
t_router, t_bridge_net, t_bridge_subnet,
is_ext_net_pod):
# NOTE(zhiyuan) after the bridge network combination, external network
# is attached to a separate router, which is created in central plugin,
# so is_ext_net_pod is not used in the current implementation, but we
# choose to keep this parameter since it's an important attribute of a
# pod and we may need to use it later.
b_client = self._get_client(b_pod['region_name'])
router_body = {'router': {'name': t_router['id'],
@ -304,61 +309,31 @@ class XManager(PeriodicTasks):
# create bottom router in target bottom pod
_, b_router_id = self.helper.prepare_bottom_element(
ctx, project_id, b_pod, t_router, 'router', router_body)
ctx, project_id, b_pod, t_router, constants.RT_ROUTER, router_body)
# handle E-W networking
# create top E-W bridge port
# create top bridge port
q_ctx = None # no need to pass neutron context when using client
t_ew_bridge_port_id = self.helper.get_bridge_interface(
ctx, q_ctx, project_id, t_pod, t_ew_bridge_net['id'],
b_router_id, None, True)
t_bridge_port_id = self.helper.get_bridge_interface(
ctx, q_ctx, project_id, t_pod, t_bridge_net['id'], b_router_id)
# create bottom E-W bridge port
t_ew_bridge_port = t_client.get_ports(ctx, t_ew_bridge_port_id)
(is_new, b_ew_bridge_port_id,
_, _) = self.helper.get_bottom_bridge_elements(
ctx, project_id, b_pod, t_ew_bridge_net, False, t_ew_bridge_subnet,
t_ew_bridge_port)
# create bottom bridge port
# if target bottom pod is hosting real external network, we create
# another bottom router and attach the bridge network as internal
# network, but this work is done by central plugin when user sets
# router gateway.
t_bridge_port = t_client.get_ports(ctx, t_bridge_port_id)
(is_new, b_bridge_port_id, b_bridge_subnet_id,
b_bridge_net_id) = self.helper.get_bottom_bridge_elements(
ctx, project_id, b_pod, t_bridge_net, True, t_bridge_subnet, None)
# attach bottom E-W bridge port to bottom router
if is_new:
# only attach bridge port the first time
b_client.action_routers(ctx, 'add_interface', b_router_id,
{'port_id': b_ew_bridge_port_id})
else:
# still need to check if the bridge port is bound
port = b_client.get_ports(ctx, b_ew_bridge_port_id)
if not port.get('device_id'):
b_client.action_routers(ctx, 'add_interface', b_router_id,
{'port_id': b_ew_bridge_port_id})
# handle N-S networking
if need_ns_bridge:
t_ns_bridge_net_name = constants.ns_bridge_net_name % project_id
t_ns_bridge_subnet_name = constants.ns_bridge_subnet_name % (
project_id)
t_ns_bridge_net = self._get_resource_by_name(
t_client, ctx, 'network', t_ns_bridge_net_name)
t_ns_bridge_subnet = self._get_resource_by_name(
t_client, ctx, 'subnet', t_ns_bridge_subnet_name)
# create bottom N-S bridge network and subnet
(_, _, b_ns_bridge_subnet_id,
b_ns_bridge_net_id) = self.helper.get_bottom_bridge_elements(
ctx, project_id, b_pod, t_ns_bridge_net, True,
t_ns_bridge_subnet, None)
# create top N-S bridge gateway port
t_ns_bridge_gateway_id = self.helper.get_bridge_interface(
ctx, q_ctx, project_id, t_pod, t_ns_bridge_net['id'],
b_router_id, None, False)
t_ns_bridge_gateway = t_client.get_ports(ctx,
t_ns_bridge_gateway_id)
# add external gateway for bottom router
# add gateway is update operation, can run multiple times
gateway_ip = t_ns_bridge_gateway['fixed_ips'][0]['ip_address']
# we attach the bridge port as router gateway
# add_gateway is update operation, which can run multiple times
gateway_ip = t_bridge_port['fixed_ips'][0]['ip_address']
b_client.action_routers(
ctx, 'add_gateway', b_router_id,
{'network_id': b_ns_bridge_net_id,
'external_fixed_ips': [{'subnet_id': b_ns_bridge_subnet_id,
{'network_id': b_bridge_net_id,
'enable_snat': False,
'external_fixed_ips': [{'subnet_id': b_bridge_subnet_id,
'ip_address': gateway_ip}]})
# attach internal port to bottom router
@ -438,107 +413,84 @@ class XManager(PeriodicTasks):
if t_int_port['network_id'] != t_net['id']:
# only handle floating ip association for the given top network
continue
if need_ns_bridge:
# create top N-S bridge interface port
t_ns_bridge_port_id = self.helper.get_bridge_interface(
ctx, q_ctx, project_id, t_pod, t_ns_bridge_net['id'], None,
b_int_port_id, False)
t_ns_bridge_port = t_client.get_ports(ctx, t_ns_bridge_port_id)
b_ext_bridge_net_id = \
db_api.get_bottom_id_by_top_id_region_name(
ctx, t_ns_bridge_net['id'], b_ext_pod['region_name'],
constants.RT_NETWORK)
if b_ext_pod['pod_id'] != b_pod['pod_id']:
# if the internal port is not located in the external network
# pod, we need to create a copied port in that pod for floating
# ip association purpose
t_int_net_id = t_int_port['network_id']
t_int_subnet_id = t_int_port['fixed_ips'][0]['subnet_id']
port_body = {
'port': {
'tenant_id': project_id,
'admin_state_up': True,
'name': 'ns_bridge_port',
'network_id': b_ext_bridge_net_id,
'fixed_ips': [{'ip_address': t_ns_bridge_port[
'name': constants.shadow_port_name % t_int_port['id'],
'network_id': t_int_net_id,
'fixed_ips': [{'ip_address': t_int_port[
'fixed_ips'][0]['ip_address']}]
}
}
_, b_ns_bridge_port_id = self.helper.prepare_bottom_element(
ctx, project_id, b_ext_pod, t_ns_bridge_port,
constants.RT_PORT, port_body)
# swap these two lines
self.helper.prepare_bottom_element(
ctx, project_id, b_ext_pod, t_int_port,
constants.RT_SD_PORT, port_body)
# create routing entries for copied network and subnet so we
# can easily find them during central network and subnet
# deletion, create_resource_mapping will catch DBDuplicateEntry
# exception and ignore it so it's safe to call this function
# multiple times
db_api.create_resource_mapping(ctx, t_int_net_id, t_int_net_id,
b_ext_pod['pod_id'], project_id,
constants.RT_SD_NETWORK)
db_api.create_resource_mapping(ctx, t_int_subnet_id,
t_int_subnet_id,
b_ext_pod['pod_id'], project_id,
constants.RT_SD_SUBNET)
self._safe_create_bottom_floatingip(
ctx, b_pod, b_client, b_ns_bridge_net_id,
t_ns_bridge_port['fixed_ips'][0]['ip_address'],
b_int_port_id)
self._safe_create_bottom_floatingip(
ctx, b_ext_pod, b_ext_client, b_ext_net_id, add_fip,
b_ns_bridge_port_id)
else:
self._safe_create_bottom_floatingip(
ctx, b_pod, b_client, b_ext_net_id, add_fip,
ctx, b_pod, b_ext_client, b_ext_net_id, add_fip,
b_int_port_id)
for del_fip in del_fips:
fip = b_ip_fip_map[del_fip]
if not fip['port_id']:
b_ext_client.delete_floatingips(ctx, fip['id'])
continue
if need_ns_bridge:
b_ns_bridge_port = b_ext_client.get_ports(ctx, fip['port_id'])
entries = core.query_resource(
ctx, models.ResourceRouting,
[{'key': 'bottom_id', 'comparator': 'eq',
'value': b_ns_bridge_port['id']},
{'key': 'pod_id', 'comparator': 'eq',
'value': b_ext_pod['pod_id']}], [])
t_ns_bridge_port_id = entries[0]['top_id']
top_entries = core.query_resource(
ctx, models.ResourceRouting,
[{'key': 'bottom_id', 'comparator': 'eq',
'value': t_ns_bridge_port_id},
{'key': 'pod_id', 'comparator': 'eq',
'value': t_pod['pod_id']}], [])
t_ns_bridge_port_name = top_entries[0]['top_id']
b_int_fips = b_client.list_floatingips(
ctx,
[{'key': 'floating_ip_address',
'comparator': 'eq',
'value': b_ns_bridge_port['fixed_ips'][0]['ip_address']},
{'key': 'floating_network_id',
'comparator': 'eq',
'value': b_ns_bridge_net_id}])
if b_int_fips:
b_client.delete_floatingips(ctx, b_int_fips[0]['id'])
# for bridge port, we have two resource routing entries, one
# for bridge port in top pod, another for bridge port in bottom
# pod. calling t_client.delete_ports will delete bridge port in
# bottom pod as well as routing entry for it, but we also need
# to remove routing entry for bridge port in top pod, bridge
# network will be deleted when deleting router
# first we update the routing entry to set bottom_id to None
# and expire the entry, so if we succeed to delete the bridge
# port next, this expired entry will be deleted; otherwise, we
# fail to delete the bridge port, when the port is accessed via
# lock_handle module, that module will find the port and update
# the entry
if b_ext_pod['pod_id'] != b_pod['pod_id'] and fip['port_id']:
# expire the routing entry for copy port
with ctx.session.begin():
core.update_resources(
ctx, models.ResourceRouting,
[{'key': 'bottom_id', 'comparator': 'eq',
'value': t_ns_bridge_port_id},
{'key': 'pod_id', 'comparator': 'eq',
'value': t_pod['pod_id']}],
'value': fip['port_id']},
{'key': 'resource_type', 'comparator': 'eq',
'value': constants.RT_SD_PORT}],
{'bottom_id': None,
'created_at': constants.expire_time,
'updated_at': constants.expire_time})
# delete bridge port
t_client.delete_ports(ctx, t_ns_bridge_port_id)
# delete copy port
b_ext_client.delete_ports(ctx, fip['port_id'])
# delete the expired entry, even if this deletion fails, we
# still have a chance that lock_handle module will delete it
with ctx.session.begin():
core.delete_resources(ctx, models.ResourceRouting,
[{'key': 'top_id',
'comparator': 'eq',
'value': t_ns_bridge_port_name}])
'value': fip['port_id']},
{'key': 'resource_type',
'comparator': 'eq',
'value': constants.RT_SD_PORT}])
# delete port before floating ip disassociation, copy
# network and copy subnet are deleted during central
# network and subnet deletion
b_ext_client.delete_floatingips(ctx, fip['id'])
# we first delete the internal port then delete the floating
# ip. during the deletion of the internal port, the floating
# ip will be disassociated automatically.
# the reason we delete the internal port first is that if we
# succeed to delete the internal port but fail to delete the
# floating ip, in the next run, we can still find the floating
# ip and try to delete it. but if we delete the floating ip
# first, after we fail to delete the internal port, it's not
# easy for us to find the internal port again because we cannot
# find the internal port id the floating ip body
@_job_handle(constants.JT_ROUTER_SETUP)
def setup_bottom_router(self, ctx, payload):
@ -570,12 +522,12 @@ class XManager(PeriodicTasks):
b_pod = db_api.get_pod(ctx, b_pod_id)
t_ew_bridge_net_name = constants.ew_bridge_net_name % project_id
t_ew_bridge_subnet_name = constants.ew_bridge_subnet_name % project_id
t_ew_bridge_net = self._get_resource_by_name(t_client, ctx, 'network',
t_ew_bridge_net_name)
t_ew_bridge_subnet = self._get_resource_by_name(
t_client, ctx, 'subnet', t_ew_bridge_subnet_name)
t_bridge_net_name = constants.bridge_net_name % project_id
t_bridge_subnet_name = constants.bridge_subnet_name % project_id
t_bridge_net = self._get_resource_by_name(t_client, ctx, 'network',
t_bridge_net_name)
t_bridge_subnet = self._get_resource_by_name(
t_client, ctx, 'subnet', t_bridge_subnet_name)
ext_nets = t_client.list_networks(ctx,
filters=[{'key': 'router:external',
@ -585,48 +537,79 @@ class XManager(PeriodicTasks):
[ext_net[AZ_HINTS][0] for ext_net in ext_nets])
if not ext_net_region_names:
need_ns_bridge = False
is_ext_net_pod = False
elif b_pod['region_name'] in ext_net_region_names:
need_ns_bridge = False
is_ext_net_pod = True
else:
need_ns_bridge = True
is_ext_net_pod = False
self._setup_router_one_pod(ctx, t_pod, b_pod, t_client, t_net,
t_router, t_ew_bridge_net,
t_ew_bridge_subnet, need_ns_bridge)
t_router, t_bridge_net,
t_bridge_subnet, is_ext_net_pod)
self.xjob_handler.configure_extra_routes(ctx, t_router_id)
@_job_handle(constants.JT_ROUTER)
def configure_extra_routes(self, ctx, payload):
t_router_id = payload[constants.JT_ROUTER]
t_client = self._get_client()
t_router = t_client.get_routers(ctx, t_router_id)
if not t_router:
return
if t_router.get('external_gateway_info'):
t_ext_net_id = t_router['external_gateway_info']['network_id']
else:
t_ext_net_id = None
non_vm_port_types = ['network:router_interface',
'network:router_gateway',
'network:dhcp']
non_vm_port_types = [q_constants.DEVICE_OWNER_ROUTER_INTF,
q_constants.DEVICE_OWNER_ROUTER_GW,
q_constants.DEVICE_OWNER_DHCP]
ew_attached_port_types = [q_constants.DEVICE_OWNER_ROUTER_INTF,
q_constants.DEVICE_OWNER_ROUTER_GW]
ns_attached_port_types = q_constants.DEVICE_OWNER_ROUTER_INTF
b_pods, b_router_ids = zip(*db_api.get_bottom_mappings_by_top_id(
ctx, t_router_id, constants.RT_ROUTER))
mappings = db_api.get_bottom_mappings_by_top_id(ctx, t_router_id,
constants.RT_ROUTER)
if not mappings:
b_pods, b_router_ids = [], []
else:
b_pods, b_router_ids = map(list, zip(*mappings))
ns_mappings = db_api.get_bottom_mappings_by_top_id(
ctx, t_router_id, constants.RT_NS_ROUTER)
b_ns_pdd, b_ns_router_id = None, None
if ns_mappings:
b_ns_pdd, b_ns_router_id = ns_mappings[0]
b_pods.append(b_ns_pdd)
b_router_ids.append(b_ns_router_id)
router_bridge_ip_map = {}
router_ew_bridge_ip_map = {}
router_ns_bridge_ip_map = {}
router_ips_map = {}
for i, b_pod in enumerate(b_pods):
bottom_client = self._get_client(region_name=b_pod['region_name'])
is_ns_router = b_router_ids[i] == b_ns_router_id
bottom_client = self._get_client(b_pod['region_name'])
if is_ns_router:
device_owner_filter = ns_attached_port_types
else:
device_owner_filter = ew_attached_port_types
b_interfaces = bottom_client.list_ports(
ctx, filters=[{'key': 'device_id',
'comparator': 'eq',
'value': b_router_ids[i]},
{'key': 'device_owner',
'comparator': 'eq',
'value': 'network:router_interface'}])
'value': device_owner_filter}])
router_ips_map[b_router_ids[i]] = {}
for b_interface in b_interfaces:
ip = b_interface['fixed_ips'][0]['ip_address']
ew_bridge_cidr = CONF.client.ew_bridge_cidr
ns_bridge_cidr = CONF.client.ns_bridge_cidr
if netaddr.IPAddress(ip) in netaddr.IPNetwork(ew_bridge_cidr):
router_bridge_ip_map[b_router_ids[i]] = ip
continue
if netaddr.IPAddress(ip) in netaddr.IPNetwork(ns_bridge_cidr):
bridge_cidr = CONF.client.bridge_cidr
if netaddr.IPAddress(ip) in netaddr.IPNetwork(bridge_cidr):
if is_ns_router:
# this ip is the default gateway ip for north-south
# networking
router_ns_bridge_ip_map[b_router_ids[i]] = ip
else:
# this ip is the next hop for east-west networking
router_ew_bridge_ip_map[b_router_ids[i]] = ip
continue
b_net_id = b_interface['network_id']
b_subnet = bottom_client.get_subnets(
@ -641,9 +624,11 @@ class XManager(PeriodicTasks):
'ip_address'] for vm_port in b_vm_ports]
router_ips_map[b_router_ids[i]][b_subnet['cidr']] = ips
# handle extra routes for east-west traffic
for i, b_router_id in enumerate(b_router_ids):
bottom_client = self._get_client(
region_name=b_pods[i]['region_name'])
if b_router_id == b_ns_router_id:
continue
bottom_client = self._get_client(b_pods[i]['region_name'])
extra_routes = []
if not router_ips_map[b_router_id]:
bottom_client.update_routers(
@ -656,12 +641,49 @@ class XManager(PeriodicTasks):
if cidr in router_ips_map[b_router_id]:
continue
for ip in ips:
route = {'nexthop': router_ew_bridge_ip_map[router_id],
'destination': ip + '/32'}
extra_routes.append(route)
if router_ns_bridge_ip_map and t_ext_net_id:
extra_routes.append(
{'nexthop': router_bridge_ip_map[router_id],
'destination': ip + '/32'})
{'nexthop': router_ns_bridge_ip_map.values()[0],
'destination': constants.DEFAULT_DESTINATION})
bottom_client.update_routers(
ctx, b_router_id, {'router': {'routes': extra_routes}})
if not b_ns_router_id:
# router for north-south networking not exist, skip extra routes
# configuration for north-south router
return
if not t_ext_net_id:
# router not attached to external gateway but router for north-
# south networking exists, clear the extra routes
bottom_client = self._get_client(pod_name=b_ns_pdd['pod_name'])
bottom_client.update_routers(
ctx, b_ns_router_id, {'router': {'routes': []}})
return
# handle extra routes for north-south router
ip_bridge_ip_map = {}
for router_id, cidr_ips_map in router_ips_map.iteritems():
if router_id not in router_ew_bridge_ip_map:
continue
for cidr, ips in cidr_ips_map.iteritems():
for ip in ips:
nexthop = router_ew_bridge_ip_map[router_id]
destination = ip + '/32'
ip_bridge_ip_map[destination] = nexthop
bottom_client = self._get_client(pod_name=b_ns_pdd['pod_name'])
extra_routes = []
for fixed_ip in ip_bridge_ip_map:
extra_routes.append(
{'nexthop': ip_bridge_ip_map[fixed_ip],
'destination': fixed_ip})
bottom_client.update_routers(
ctx, b_ns_router_id, {'router': {'routes': extra_routes}})
@_job_handle(constants.JT_PORT_DELETE)
def delete_server_port(self, ctx, payload):
b_pod_id, b_port_id = payload[constants.JT_PORT_DELETE].split('#')
@ -727,14 +749,10 @@ class XManager(PeriodicTasks):
subnets = top_client.list_subnets(
ctx, [{'key': 'tenant_id', 'comparator': 'eq',
'value': project_id}])
ew_bridge_ip_net = netaddr.IPNetwork(
CONF.client.ew_bridge_cidr)
ns_bridge_ip_net = netaddr.IPNetwork(
CONF.client.ns_bridge_cidr)
bridge_ip_net = netaddr.IPNetwork(CONF.client.bridge_cidr)
for subnet in subnets:
ip_net = netaddr.IPNetwork(subnet['cidr'])
if ip_net in ew_bridge_ip_net or (
ip_net in ns_bridge_ip_net):
if ip_net in bridge_ip_net:
continue
# leave sg_id empty here
new_b_rules.append(