From 69e9888b0159ba3c9af09e1eceb48df76e4be839 Mon Sep 17 00:00:00 2001 From: Ryan Tidwell Date: Tue, 29 May 2018 12:40:30 -0500 Subject: [PATCH] Implement DVR-aware fixed IP lookups This change adds DVR-aware announcements for routable fixed IP's to be sent, thereby routing traffic directly to the appropriate compute node instead of the centralized router on the network node. Change-Id: I3aecdd7979dba97dab12a6550655c90a57f56cb3 Partial-Bug: #1775250 --- neutron_dynamic_routing/db/bgp_db.py | 123 +++++++++- .../services/bgp/bgp_plugin.py | 34 +++ .../tests/unit/db/test_bgp_db.py | 225 +++++++++++++++++- .../unit/services/bgp/test_bgp_plugin.py | 2 + ...-aware-announcements-24bfcb8fee87161d.yaml | 8 + 5 files changed, 377 insertions(+), 15 deletions(-) create mode 100644 releasenotes/notes/dvr-aware-announcements-24bfcb8fee87161d.yaml diff --git a/neutron_dynamic_routing/db/bgp_db.py b/neutron_dynamic_routing/db/bgp_db.py index 99261825..35834362 100644 --- a/neutron_dynamic_routing/db/bgp_db.py +++ b/neutron_dynamic_routing/db/bgp_db.py @@ -22,6 +22,9 @@ from neutron.db.models import address_scope as address_scope_db from neutron.db.models import l3 as l3_db from neutron.db.models import l3_attrs as l3_attrs_db from neutron.db import models_v2 +from neutron.objects import ports +from neutron.objects import subnet as subnet_obj +from neutron.objects import subnetpool as subnetpool_obj from neutron.plugins.ml2 import models as ml2_models from neutron_lib.api import validators @@ -46,6 +49,7 @@ from neutron_dynamic_routing.extensions import bgp as bgp_ext DEVICE_OWNER_ROUTER_GW = lib_consts.DEVICE_OWNER_ROUTER_GW DEVICE_OWNER_ROUTER_INTF = lib_consts.DEVICE_OWNER_ROUTER_INTF +DEVICE_OWNER_DVR_INTERFACE = lib_consts.DEVICE_OWNER_DVR_INTERFACE class BgpSpeakerPeerBinding(model_base.BASEV2): @@ -477,7 +481,11 @@ class BgpDbMixin(common_db.CommonDbMixin): dvr_fip_routes = self._get_dvr_fip_host_routes_by_bgp_speaker( context, bgp_speaker_id) - return itertools.chain(fip_routes, net_routes, dvr_fip_routes) + dvr_fixedip_routes = self._get_dvr_fixed_ip_routes_by_bgp_speaker( + context, + bgp_speaker_id) + return itertools.chain(fip_routes, net_routes, dvr_fip_routes, + dvr_fixedip_routes) def get_routes_by_bgp_speaker_binding(self, context, bgp_speaker_id, network_id): @@ -669,12 +677,15 @@ class BgpDbMixin(common_db.CommonDbMixin): def _get_gateway_query(self, context, bgp_speaker_id): BgpBinding = BgpSpeakerNetworkBinding + AddressScope = address_scope_db.AddressScope ML2PortBinding = ml2_models.PortBinding IpAllocation = models_v2.IPAllocation Port = models_v2.Port - gw_query = context.session.query(Port.network_id, - ML2PortBinding.host, - IpAllocation.ip_address) + gw_query = context.session.query( + Port.network_id, + ML2PortBinding.host, + IpAllocation.ip_address, + AddressScope.id.label('address_scope_id')) #Subquery for FIP agent gateway ports gw_query = gw_query.filter( @@ -682,6 +693,9 @@ class BgpDbMixin(common_db.CommonDbMixin): IpAllocation.port_id == Port.id, IpAllocation.subnet_id == models_v2.Subnet.id, models_v2.Subnet.ip_version == 4, + models_v2.Subnet.subnetpool_id == models_v2.SubnetPool.id, + models_v2.SubnetPool.address_scope_id == AddressScope.id, + Port.network_id == models_v2.Subnet.network_id, Port.device_owner == lib_consts.DEVICE_OWNER_AGENT_GW, Port.network_id == BgpBinding.network_id, BgpBinding.bgp_speaker_id == bgp_speaker_id, @@ -703,6 +717,49 @@ class BgpDbMixin(common_db.CommonDbMixin): BgpBinding.bgp_speaker_id == bgp_speaker_id) return fip_query + def _get_dvr_fixed_ip_query(self, context, bgp_speaker_id): + AddressScope = address_scope_db.AddressScope + ML2PortBinding = ml2_models.PortBinding + Port = models_v2.Port + IpAllocation = models_v2.IPAllocation + + fixed_ip_query = context.session.query( + ML2PortBinding.host, + IpAllocation.ip_address, + IpAllocation.subnet_id, + AddressScope.id.label('address_scope_id')) + fixed_ip_query = fixed_ip_query.filter( + Port.id == ML2PortBinding.port_id, + IpAllocation.port_id == Port.id, + Port.device_owner.startswith( + lib_consts.DEVICE_OWNER_COMPUTE_PREFIX), + IpAllocation.subnet_id == models_v2.Subnet.id, + models_v2.Subnet.subnetpool_id == models_v2.SubnetPool.id, + AddressScope.id == models_v2.SubnetPool.address_scope_id) + return fixed_ip_query + + def _get_dvr_fixed_ip_routes_by_bgp_speaker(self, context, + bgp_speaker_id): + with db_api.CONTEXT_READER.using(context): + gw_query = self._get_gateway_query(context, bgp_speaker_id) + fixed_query = self._get_dvr_fixed_ip_query(context, + bgp_speaker_id) + join_query = self._join_fixed_by_host_binding_to_agent_gateway( + context, + fixed_query.subquery(), + gw_query.subquery()) + return self._host_route_list_from_tuples(join_query.all()) + + def _join_fixed_by_host_binding_to_agent_gateway(self, context, + fixed_subq, gw_subq): + join_query = context.session.query(fixed_subq.c.ip_address, + gw_subq.c.ip_address) + and_cond = and_( + gw_subq.c.host == fixed_subq.c.host, + gw_subq.c.address_scope_id == fixed_subq.c.address_scope_id) + + return join_query.join(gw_subq, and_cond) + def _get_dvr_fip_host_routes_by_bgp_speaker(self, context, bgp_speaker_id): router_attrs = l3_attrs_db.RouterExtraAttributes @@ -1084,3 +1141,61 @@ class BgpDbMixin(common_db.CommonDbMixin): return except sa_exc.MultipleResultsFound: return + + def get_external_networks_for_port(self, ctx, port, + match_address_scopes=True): + with db_api.CONTEXT_READER.using(ctx): + # Retrieve address scope info for the supplied port + port_fixed_ips = port.get('fixed_ips') + if not port_fixed_ips: + return [] + subnets_filter = {'id': [x['subnet_id'] for x in port_fixed_ips]} + port_subnets = subnet_obj.Subnet.get_objects(ctx, **subnets_filter) + port_subnetpools = subnetpool_obj.SubnetPool.get_objects( + ctx, id=[x.subnetpool_id for x in port_subnets]) + port_scopes = set([x.address_scope_id for x in port_subnetpools]) + if match_address_scopes and len(port_scopes) == 0: + return [] + + # Get all router IDs with an interface on the given port's network + router_iface_filters = {'device_owner': + [DEVICE_OWNER_ROUTER_INTF, + DEVICE_OWNER_DVR_INTERFACE], + 'network_id': port['network_id']} + router_ids = [x.device_id for x in ports.Port.get_objects( + ctx, **router_iface_filters)] + + # Retrieve the gateway ports for the identified routers + gw_port_filters = {'device_owner': DEVICE_OWNER_ROUTER_GW, + 'device_id': router_ids} + gw_ports = ports.Port.get_objects(ctx, **gw_port_filters) + + # If we don't need to match address scopes, return here + if not match_address_scopes: + return list(set([x.network_id for x in gw_ports])) + + # Retrieve address scope info for associated gateway networks + gw_fixed_ips = [] + for gw_port in gw_ports: + gw_fixed_ips.extend(gw_port.fixed_ips) + gw_subnet_filters = {'id': [x.subnet_id for x in gw_fixed_ips]} + gw_subnets = subnet_obj.Subnet.get_objects(ctx, + **gw_subnet_filters) + ext_net_subnetpool_map = {} + for gw_subnet in gw_subnets: + ext_net_id = gw_subnet.network_id + ext_pool = subnetpool_obj.SubnetPool.get_object( + ctx, id=gw_subnet.subnetpool_id) + ext_scope_set = ext_net_subnetpool_map.get(ext_net_id, set()) + ext_scope_set.add(ext_pool.address_scope_id) + ext_net_subnetpool_map[ext_net_id] = ext_scope_set + + ext_nets = [] + + # Match address scopes between port and gateway network(s) + for net in ext_net_subnetpool_map.keys(): + ext_scopes = ext_net_subnetpool_map[net] + if ext_scopes.issubset(port_scopes): + ext_nets.append(net) + + return ext_nets diff --git a/neutron_dynamic_routing/services/bgp/bgp_plugin.py b/neutron_dynamic_routing/services/bgp/bgp_plugin.py index aebeba53..bae0f480 100644 --- a/neutron_dynamic_routing/services/bgp/bgp_plugin.py +++ b/neutron_dynamic_routing/services/bgp/bgp_plugin.py @@ -14,6 +14,7 @@ from netaddr import IPAddress +from neutron_lib.api.definitions import portbindings from neutron_lib.callbacks import events from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources @@ -94,6 +95,9 @@ class BgpPlugin(service_base.ServicePluginBase, registry.subscribe(self.router_gateway_callback, resources.ROUTER_GATEWAY, events.AFTER_DELETE) + registry.subscribe(self.port_callback, + resources.PORT, + events.AFTER_UPDATE) def get_bgp_speakers(self, context, filters=None, fields=None, sorts=None, limit=None, marker=None, @@ -343,6 +347,36 @@ class BgpPlugin(service_base.ServicePluginBase, self.stop_route_advertisements(ctx, self._bgp_rpc, speaker.id, routes) + def port_callback(self, resource, event, trigger, **kwargs): + if event != events.AFTER_UPDATE: + return + + original_port = kwargs['original_port'] + updated_port = kwargs['port'] + if not updated_port.get('fixed_ips'): + return + + original_host = original_port.get(portbindings.HOST_ID) + updated_host = updated_port.get(portbindings.HOST_ID) + device_owner = updated_port.get('device_owner') + + # if host in the port binding has changed, update next-hops + if original_host != updated_host and bool('compute:' in device_owner): + ctx = context.get_admin_context() + with ctx.session.begin(subtransactions=True): + ext_nets = self.get_external_networks_for_port(ctx, + updated_port) + for ext_net in ext_nets: + bgp_speakers = ( + self._get_bgp_speaker_ids_by_binding_network( + ctx, ext_nets)) + + # Refresh any affected BGP speakers + for bgp_speaker in bgp_speakers: + routes = self.get_advertised_routes(ctx, bgp_speaker) + self.start_route_advertisements(ctx, self._bgp_rpc, + bgp_speaker, routes) + def _next_hops_from_gateway_ips(self, gw_ips): if gw_ips: return {IPAddress(ip).version: ip for ip in gw_ips} diff --git a/neutron_dynamic_routing/tests/unit/db/test_bgp_db.py b/neutron_dynamic_routing/tests/unit/db/test_bgp_db.py index 0258cba1..f09be469 100644 --- a/neutron_dynamic_routing/tests/unit/db/test_bgp_db.py +++ b/neutron_dynamic_routing/tests/unit/db/test_bgp_db.py @@ -127,27 +127,35 @@ class BgpEntityCreationMixin(object): tenant_prefix='192.168.0.0/16', address_scope=None, distributed=False, - ha=False): - prefixes = [gw_prefix, tenant_prefix] + ha=False, + ext_net_use_addr_scope=True, + tenant_net_use_addr_scope=True): gw_ip_net = netaddr.IPNetwork(gw_prefix) tenant_ip_net = netaddr.IPNetwork(tenant_prefix) - subnetpool_args = {'tenant_id': tenant_id, - 'name': 'bgp-pool'} - if address_scope: - subnetpool_args['address_scope_id'] = address_scope['id'] + ext_pool_args = {'tenant_id': tenant_id, + 'name': 'bgp-pool'} + tenant_pool_args = ext_pool_args.copy() + + if address_scope and ext_net_use_addr_scope: + ext_pool_args['address_scope_id'] = address_scope['id'] + + if address_scope and tenant_net_use_addr_scope: + tenant_pool_args['address_scope_id'] = address_scope['id'] with self.gw_network(external=True) as ext_net,\ self.network() as int_net,\ - self.subnetpool(prefixes, **subnetpool_args) as pool: - subnetpool_id = pool['subnetpool']['id'] + self.subnetpool([gw_prefix], **ext_pool_args) as ext_pool,\ + self.subnetpool([tenant_prefix], **tenant_pool_args) as int_pool: + ext_subnetpool_id = ext_pool['subnetpool']['id'] + int_subnetpool_id = int_pool['subnetpool']['id'] gw_net_id = ext_net['network']['id'] with self.subnet(ext_net, cidr=gw_prefix, - subnetpool_id=subnetpool_id, + subnetpool_id=ext_subnetpool_id, ip_version=gw_ip_net.version),\ self.subnet(int_net, cidr=tenant_prefix, - subnetpool_id=subnetpool_id, + subnetpool_id=int_subnetpool_id, ip_version=tenant_ip_net.version) as int_subnet: ext_gw_info = {'network_id': gw_net_id} with self.router(external_gateway_info=ext_gw_info, @@ -895,6 +903,7 @@ class BgpTests(BgpEntityCreationMixin): 'port_id': fixed_port['id']}} fip = self.l3plugin.create_floatingip(self.context, fip_data) fip_prefix = fip['floating_ip_address'] + '/32' + fixed_prefix = fixed_port['fixed_ips'][0]['ip_address'] + '/32' with self.bgp_speaker(4, 1234, networks=[gw_net_id]) as speaker: bgp_speaker_id = speaker['id'] routes = self.bgp_plugin.get_routes_by_bgp_speaker_id( @@ -903,9 +912,10 @@ class BgpTests(BgpEntityCreationMixin): routes = list(routes) cvr_gw_ip = ext_gw_info['external_fixed_ips'][0]['ip_address'] dvr_gw_ip = fip_gw['fixed_ips'][0]['ip_address'] - self.assertEqual(2, len(routes)) + self.assertEqual(3, len(routes)) tenant_route_verified = False fip_route_verified = False + fixed_ip_route_verified = False for route in routes: if route['destination'] == tenant_prefix: self.assertEqual(cvr_gw_ip, route['next_hop']) @@ -913,8 +923,12 @@ class BgpTests(BgpEntityCreationMixin): if route['destination'] == fip_prefix: self.assertEqual(dvr_gw_ip, route['next_hop']) fip_route_verified = True + if route['destination'] == fixed_prefix: + self.assertEqual(dvr_gw_ip, route['next_hop']) + fixed_ip_route_verified = True self.assertTrue(tenant_route_verified) self.assertTrue(fip_route_verified) + self.assertTrue(fixed_ip_route_verified) def test__get_dvr_fip_host_routes_by_binding(self): gw_prefix = '172.16.10.0/24' @@ -1302,6 +1316,195 @@ class BgpTests(BgpEntityCreationMixin): def test__get_fip_next_hop_dvr(self): self._test__get_fip_next_hop(distributed=True) + def _test__get_dvr_fixed_ip_routes_by_bgp_speaker(self, + ext_use_scope, + tenant_use_scope): + gw_prefix = '172.16.10.0/24' + tenant_prefix = '10.10.10.0/24' + tenant_id = _uuid() + scope_data = {'tenant_id': tenant_id, 'ip_version': 4, + 'shared': True, 'name': 'bgp-scope'} + scope = self.plugin.create_address_scope( + self.context, + {'address_scope': scope_data}) + with self.router_with_external_and_tenant_networks( + tenant_id=tenant_id, + gw_prefix=gw_prefix, + tenant_prefix=tenant_prefix, + address_scope=scope, + distributed=True, + ext_net_use_addr_scope=ext_use_scope, + tenant_net_use_addr_scope=tenant_use_scope) as res: + router, ext_net, int_net = res + gw_net_id = ext_net['network']['id'] + tenant_net_id = int_net['network']['id'] + fixed_port_data = {'port': + {'name': 'test', + 'network_id': tenant_net_id, + 'tenant_id': tenant_id, + 'admin_state_up': True, + 'device_id': _uuid(), + 'device_owner': 'compute:nova', + 'mac_address': n_const.ATTR_NOT_SPECIFIED, + 'fixed_ips': n_const.ATTR_NOT_SPECIFIED, + portbindings.HOST_ID: 'test-host'}} + fixed_port = self.plugin.create_port(self.context, + fixed_port_data) + fixed_ip_prefix = fixed_port['fixed_ips'][0]['ip_address'] + '/32' + self.plugin.create_or_update_agent(self.context, + {'agent_type': 'L3 agent', + 'host': 'test-host', + 'binary': 'neutron-l3-agent', + 'topic': 'test'}) + fip_gw = self.l3plugin.create_fip_agent_gw_port_if_not_exists( + self.context, + gw_net_id, + 'test-host') + with self.bgp_speaker(4, 1234, networks=[gw_net_id]) as speaker: + bgp_speaker_id = speaker['id'] + routes = self.bgp_plugin._get_dvr_fixed_ip_routes_by_bgp_speaker( # noqa + self.context, + bgp_speaker_id) + routes = list(routes) + dvr_gw_ip = fip_gw['fixed_ips'][0]['ip_address'] + if ext_use_scope and tenant_use_scope: + self.assertEqual(1, len(routes)) + self.assertEqual(dvr_gw_ip, routes[0]['next_hop']) + self.assertEqual(fixed_ip_prefix, routes[0]['destination']) + else: + self.assertEqual(0, len(routes)) + + def test__get_dvr_fixed_ip_routes_by_bgp_speaker_same_scope(self): + self._test__get_dvr_fixed_ip_routes_by_bgp_speaker(True, True) + + def test__get_dvr_fixed_ip_routes_by_bgp_speaker_different_scope(self): + self._test__get_dvr_fixed_ip_routes_by_bgp_speaker(True, False) + self._test__get_dvr_fixed_ip_routes_by_bgp_speaker(False, True) + + def test__get_dvr_fixed_ip_routes_by_bgp_speaker_no_scope(self): + self._test__get_dvr_fixed_ip_routes_by_bgp_speaker(False, False) + + def test_get_external_networks_for_port_same_address_scope_v4(self): + tenant_id = _uuid() + scope_data = {'tenant_id': tenant_id, 'ip_version': 4, + 'shared': True, 'name': 'bgp-scope'} + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + True, True) + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + True, True, False) + + def test_get_external_networks_for_port_different_address_scope_v4(self): + tenant_id = _uuid() + scope_data = {'tenant_id': tenant_id, 'ip_version': 4, + 'shared': True, 'name': 'bgp-scope'} + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + True, False) + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + True, False, False) + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + False, True) + self._test_get_external_networks_for_port('172.10.2.0/24', + '10.0.0.0/24', + scope_data, tenant_id, + False, True, False) + + def test_get_external_networks_for_port_same_address_scope_v6(self): + tenant_id = _uuid() + scope_data = {'tenant_id': tenant_id, 'ip_version': 6, + 'shared': True, 'name': 'bgp-scope'} + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + True, True) + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + True, True, False) + + def test_get_external_networks_for_port_different_address_scope_v6(self): + tenant_id = _uuid() + scope_data = {'tenant_id': tenant_id, 'ip_version': 6, + 'shared': True, 'name': 'bgp-scope'} + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + True, False) + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + True, False, False) + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + False, True) + self._test_get_external_networks_for_port('2001:1234:1234::/64', + '2001:1234:4321::/64', + scope_data, tenant_id, + False, True, False) + + def _test_get_external_networks_for_port(self, gw_prefix, tenant_prefix, + scope_data, tenant_id, + external_use_address_scope, + project_use_address_scope, + match_address_scopes=True): + scope = None + if scope_data: + scope = self.plugin.create_address_scope( + self.context, + {'address_scope': scope_data}) + + with self.router_with_external_and_tenant_networks( + tenant_id=tenant_id, + gw_prefix=gw_prefix, + tenant_prefix=tenant_prefix, + address_scope=scope, + distributed=True, + ext_net_use_addr_scope=external_use_address_scope, + tenant_net_use_addr_scope=project_use_address_scope) as res: + router, ext_net, int_net = res + gw_net_id = ext_net['network']['id'] + tenant_net_id = int_net['network']['id'] + fixed_port_data = {'port': + {'name': 'test', + 'network_id': tenant_net_id, + 'tenant_id': tenant_id, + 'admin_state_up': True, + 'device_id': _uuid(), + 'device_owner': 'compute:nova', + 'mac_address': n_const.ATTR_NOT_SPECIFIED, + 'fixed_ips': n_const.ATTR_NOT_SPECIFIED, + portbindings.HOST_ID: 'test-host'}} + fixed_port = self.plugin.create_port(self.context, + fixed_port_data) + self.plugin.create_or_update_agent(self.context, + {'agent_type': 'L3 agent', + 'host': 'test-host', + 'binary': 'neutron-l3-agent', + 'topic': 'test'}) + ext_nets = self.bgp_plugin.get_external_networks_for_port( + self.context, + fixed_port, + match_address_scopes=match_address_scopes) + + if not match_address_scopes: + self.assertEqual(1, len(ext_nets)) + self.assertEqual(gw_net_id, ext_nets[0]) + elif external_use_address_scope and project_use_address_scope: + self.assertEqual(1, len(ext_nets)) + self.assertEqual(gw_net_id, ext_nets[0]) + else: + self.assertEqual(0, len(ext_nets)) + class Ml2BgpTests(test_plugin.Ml2PluginV2TestCase, BgpTests): diff --git a/neutron_dynamic_routing/tests/unit/services/bgp/test_bgp_plugin.py b/neutron_dynamic_routing/tests/unit/services/bgp/test_bgp_plugin.py index 03b4a87a..20d0e455 100644 --- a/neutron_dynamic_routing/tests/unit/services/bgp/test_bgp_plugin.py +++ b/neutron_dynamic_routing/tests/unit/services/bgp/test_bgp_plugin.py @@ -66,6 +66,8 @@ class TestBgpPlugin(base.BaseTestCase): resources.ROUTER_GATEWAY, events.AFTER_CREATE), mock.call(plugin.router_gateway_callback, resources.ROUTER_GATEWAY, events.AFTER_DELETE), + mock.call(plugin.port_callback, + resources.PORT, events.AFTER_UPDATE), ] self.assertEqual(subscribe.call_args_list, expected_calls) diff --git a/releasenotes/notes/dvr-aware-announcements-24bfcb8fee87161d.yaml b/releasenotes/notes/dvr-aware-announcements-24bfcb8fee87161d.yaml new file mode 100644 index 00000000..00804cac --- /dev/null +++ b/releasenotes/notes/dvr-aware-announcements-24bfcb8fee87161d.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + DVR-aware BGP announcements are now supported for IPv4. Host routes + for instances are announced as /32 host routes, using the appropriate + floating IP gateway port on the host as the next-hop. This allows + network traffic to bypass the centralized router on the network + node and ingress/egress directly on the compute node.