From f63366e615e2d4e799fe8072b450bce96f179c9e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Thu, 4 Dec 2014 16:15:43 -0800 Subject: [PATCH] L3 Agent support for routers with HA and DVR The main difference for DVR HA routers is where the VRRP/keepalived logic is run and which ports fall in the HA domain for DVR. Instead of running in the qrouter namespace, keepalived will run inside the snat-namespace. Therefore only snat ports will fall under the control of the HA domain. Partial-Bug: #1365473 Change-Id: If2962580397d39f72fd1fbbc1188a6958f00ff0c Co-Authored-By: Michael Smith Co-Authored-By: Hardik Italia Co-Authored-By: Adolfo Duarte Co-Authored-By: John Schwarz --- neutron/agent/l3/agent.py | 13 +- neutron/agent/l3/dvr_edge_ha_router.py | 129 +++++++++++ neutron/agent/l3/dvr_edge_router.py | 25 ++- neutron/agent/l3/dvr_local_router.py | 2 + neutron/agent/l3/ha.py | 6 +- neutron/agent/l3/ha_router.py | 48 ++-- .../tests/functional/agent/test_l3_agent.py | 208 +++++++++++++++--- 7 files changed, 371 insertions(+), 60 deletions(-) create mode 100644 neutron/agent/l3/dvr_edge_ha_router.py diff --git a/neutron/agent/l3/agent.py b/neutron/agent/l3/agent.py index 570a6c420f6..cbf92ca5830 100644 --- a/neutron/agent/l3/agent.py +++ b/neutron/agent/l3/agent.py @@ -25,6 +25,7 @@ from oslo_utils import timeutils from neutron.agent.common import utils as common_utils from neutron.agent.l3 import dvr +from neutron.agent.l3 import dvr_edge_ha_router from neutron.agent.l3 import dvr_edge_router as dvr_router from neutron.agent.l3 import dvr_local_router as dvr_local_router from neutron.agent.l3 import ha @@ -297,11 +298,6 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, raise Exception(msg) def _create_router(self, router_id, router): - # TODO(Carl) We need to support a router that is both HA and DVR. The - # patch that enables it will replace these lines. See bug #1365473. - if router.get('distributed') and router.get('ha'): - raise n_exc.DvrHaRouterNotSupported(router_id=router_id) - args = [] kwargs = { 'router_id': router_id, @@ -314,6 +310,13 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, if router.get('distributed'): kwargs['agent'] = self kwargs['host'] = self.host + + if router.get('distributed') and router.get('ha'): + if self.conf.agent_mode == l3_constants.L3_AGENT_MODE_DVR_SNAT: + kwargs['state_change_callback'] = self.enqueue_state_change + return dvr_edge_ha_router.DvrEdgeHaRouter(*args, **kwargs) + + if router.get('distributed'): if self.conf.agent_mode == l3_constants.L3_AGENT_MODE_DVR_SNAT: return dvr_router.DvrEdgeRouter(*args, **kwargs) else: diff --git a/neutron/agent/l3/dvr_edge_ha_router.py b/neutron/agent/l3/dvr_edge_ha_router.py new file mode 100644 index 00000000000..5e254d65dd8 --- /dev/null +++ b/neutron/agent/l3/dvr_edge_ha_router.py @@ -0,0 +1,129 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from neutron.agent.l3.dvr_edge_router import DvrEdgeRouter +from neutron.agent.l3 import dvr_snat_ns +from neutron.agent.l3.ha_router import HaRouter +from neutron.agent.l3.router_info import RouterInfo +from neutron.common import constants as l3_constants + +LOG = logging.getLogger(__name__) + + +class DvrEdgeHaRouter(DvrEdgeRouter, HaRouter): + """Router class which represents a centralized SNAT + DVR router with HA capabilities. + """ + + def __init__(self, agent, host, *args, **kwargs): + super(DvrEdgeHaRouter, self).__init__(agent, host, + *args, **kwargs) + self.enable_snat = None + self.snat_ports = None + + @property + def ha_namespace(self): + if self.snat_namespace: + return self.snat_namespace.name + return None + + def internal_network_added(self, port): + # Call RouterInfo's internal_network_added (Plugs the port, adds IP) + RouterInfo.internal_network_added(self, port) + + for subnet in port['subnets']: + self._set_subnet_arp_info(subnet['id']) + self._snat_redirect_add_from_port(port) + + if not self.get_ex_gw_port() or not self._is_this_snat_host(): + return + + sn_port = self.get_snat_port_for_internal_port(port) + if not sn_port: + return + + self._plug_ha_router_port( + sn_port, + self._get_snat_int_device_name, + dvr_snat_ns.SNAT_INT_DEV_PREFIX) + + def external_gateway_added(self, ex_gw_port, interface_name): + super(DvrEdgeHaRouter, self).external_gateway_added( + ex_gw_port, interface_name) + for port in self.get_snat_interfaces(): + snat_interface_name = self._get_snat_int_device_name(port['id']) + self._disable_ipv6_addressing_on_interface(snat_interface_name) + self._add_vips( + self.get_snat_port_for_internal_port(port), + snat_interface_name) + + self._add_gateway_vip(ex_gw_port, interface_name) + self._disable_ipv6_addressing_on_interface(interface_name) + + def external_gateway_removed(self, ex_gw_port, interface_name): + for port in self.snat_ports: + snat_interface = self._get_snat_int_device_name(port['id']) + self.driver.unplug(snat_interface, + namespace=self.ha_namespace, + prefix=l3_constants.SNAT_INT_DEV_PREFIX) + self._clear_vips(snat_interface) + super(DvrEdgeHaRouter, self)._external_gateway_removed( + ex_gw_port, interface_name) + self._clear_vips(interface_name) + + def external_gateway_updated(self, ex_gw_port, interface_name): + HaRouter.external_gateway_updated(self, ex_gw_port, interface_name) + + def initialize(self, process_monitor): + self._create_snat_namespace() + super(DvrEdgeHaRouter, self).initialize(process_monitor) + + def process(self, agent): + super(DvrEdgeHaRouter, self).process(agent) + if self.ha_port: + self.enable_keepalived() + + def delete(self, agent): + super(DvrEdgeHaRouter, self).delete(agent) + if self.snat_namespace: + self.snat_namespace.delete() + + def get_router_cidrs(self, device): + return RouterInfo.get_router_cidrs(self, device) + + def _external_gateway_added(self, ex_gw_port, interface_name, + ns_name, preserve_ips): + self._plug_external_gateway(ex_gw_port, interface_name, ns_name) + + def _is_this_snat_host(self): + return (self.agent_conf.agent_mode + == l3_constants.L3_AGENT_MODE_DVR_SNAT) + + def _dvr_internal_network_removed(self, port): + super(DvrEdgeHaRouter, self)._dvr_internal_network_removed(port) + sn_port = self.get_snat_port_for_internal_port(port, self.snat_ports) + if not sn_port: + return + self._clear_vips(self._get_snat_int_device_name(sn_port['id'])) + + def _plug_snat_port(self, port): + """Used by _create_dvr_gateway in DvrEdgeRouter.""" + interface_name = self._get_snat_int_device_name(port['id']) + self.driver.plug(port['network_id'], port['id'], + interface_name, port['mac_address'], + namespace=self.snat_namespace.name, + prefix=dvr_snat_ns.SNAT_INT_DEV_PREFIX) diff --git a/neutron/agent/l3/dvr_edge_router.py b/neutron/agent/l3/dvr_edge_router.py index 2d4bd20e31a..72534d35ca2 100644 --- a/neutron/agent/l3/dvr_edge_router.py +++ b/neutron/agent/l3/dvr_edge_router.py @@ -57,7 +57,7 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter): self.snat_namespace.name, preserve_ips=[]) - def external_gateway_removed(self, ex_gw_port, interface_name): + def _external_gateway_removed(self, ex_gw_port, interface_name): super(DvrEdgeRouter, self).external_gateway_removed(ex_gw_port, interface_name) if not self._is_this_snat_host() and not self.snat_namespace: @@ -69,8 +69,12 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter): bridge=self.agent_conf.external_network_bridge, namespace=self.snat_namespace.name, prefix=router.EXTERNAL_DEV_PREFIX) - self.snat_namespace.delete() - self.snat_namespace = None + + def external_gateway_removed(self, ex_gw_port, interface_name): + self._external_gateway_removed(ex_gw_port, interface_name) + if self.snat_namespace: + self.snat_namespace.delete() + self.snat_namespace = None def internal_network_added(self, port): super(DvrEdgeRouter, self).internal_network_added(port) @@ -115,18 +119,21 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter): self.driver.unplug(snat_interface, namespace=ns_name, prefix=prefix) + def _plug_snat_port(self, port): + interface_name = self._get_snat_int_device_name(port['id']) + self._internal_network_added( + self.snat_namespace.name, port['network_id'], + port['id'], port['fixed_ips'], + port['mac_address'], interface_name, + dvr_snat_ns.SNAT_INT_DEV_PREFIX) + def _create_dvr_gateway(self, ex_gw_port, gw_interface_name): """Create SNAT namespace.""" snat_ns = self._create_snat_namespace() # connect snat_ports to br_int from SNAT namespace for port in self.get_snat_interfaces(): # create interface_name - interface_name = self._get_snat_int_device_name(port['id']) - self._internal_network_added( - snat_ns.name, port['network_id'], - port['id'], port['fixed_ips'], - port['mac_address'], interface_name, - dvr_snat_ns.SNAT_INT_DEV_PREFIX) + self._plug_snat_port(port) self._external_gateway_added(ex_gw_port, gw_interface_name, snat_ns.name, preserve_ips=[]) self.snat_iptables_manager = iptables_manager.IptablesManager( diff --git a/neutron/agent/l3/dvr_local_router.py b/neutron/agent/l3/dvr_local_router.py index 1ecc79fc477..35a2c0af13f 100644 --- a/neutron/agent/l3/dvr_local_router.py +++ b/neutron/agent/l3/dvr_local_router.py @@ -298,7 +298,9 @@ class DvrLocalRouter(dvr_router_base.DvrRouterBase): # external_gateway port or the agent_mode. for subnet in port['subnets']: self._set_subnet_arp_info(subnet['id']) + self._snat_redirect_add_from_port(port) + def _snat_redirect_add_from_port(self, port): ex_gw_port = self.get_ex_gw_port() if not ex_gw_port: return diff --git a/neutron/agent/l3/ha.py b/neutron/agent/l3/ha.py index b66e1a37d22..bafc7e71c63 100644 --- a/neutron/agent/l3/ha.py +++ b/neutron/agent/l3/ha.py @@ -137,7 +137,11 @@ class AgentMixin(object): gateway_ips = ri._get_external_gw_ips(ri.ex_gw_port) if not ri.is_v6_gateway_set(gateway_ips): interface_name = ri.get_external_device_name(ex_gw_port_id) - ri.driver.configure_ipv6_ra(ri.ns_name, interface_name) + if ri.router.get('distributed', False): + namespace = ri.ha_namespace + else: + namespace = ri.ns_name + ri.driver.configure_ipv6_ra(namespace, interface_name) def _update_metadata_proxy(self, ri, router_id, state): if state == 'master': diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py index b041da3899c..55d4eebb961 100644 --- a/neutron/agent/l3/ha_router.py +++ b/neutron/agent/l3/ha_router.py @@ -69,6 +69,10 @@ class HaRouter(router.RouterInfo): LOG.error(_LE('Error while writing HA state for %s'), self.router_id) + @property + def ha_namespace(self): + return self.ns_name + def initialize(self, process_monitor): super(HaRouter, self).initialize(process_monitor) ha_port = self.router.get(n_consts.HA_INTERFACE_KEY) @@ -89,7 +93,7 @@ class HaRouter(router.RouterInfo): keepalived.KeepalivedConf(), process_monitor, conf_path=self.agent_conf.ha_confs_path, - namespace=self.ns_name) + namespace=self.ha_namespace) config = self.keepalived_manager.config @@ -138,19 +142,26 @@ class HaRouter(router.RouterInfo): self.ha_port['id'], interface_name, self.ha_port['mac_address'], - namespace=self.ns_name, + namespace=self.ha_namespace, prefix=HA_DEV_PREFIX) ip_cidrs = common_utils.fixed_ip_cidrs(self.ha_port['fixed_ips']) self.driver.init_l3(interface_name, ip_cidrs, - namespace=self.ns_name, + namespace=self.ha_namespace, preserve_ips=[self._get_primary_vip()]) def ha_network_removed(self): self.driver.unplug(self.get_ha_device_name(), - namespace=self.ns_name, + namespace=self.ha_namespace, prefix=HA_DEV_PREFIX) self.ha_port = None + def _add_vips(self, port, interface_name): + for ip_cidr in common_utils.fixed_ip_cidrs(port['fixed_ips']): + try: + self._add_vip(ip_cidr, interface_name) + except keepalived.VIPDuplicateAddressException: + LOG.debug("%s has already been added to keepalive", ip_cidr) + def _add_vip(self, ip_cidr, interface, scope=None): instance = self._get_keepalived_instance() instance.add_vip(ip_cidr, interface, scope) @@ -223,7 +234,7 @@ class HaRouter(router.RouterInfo): a VIP to keepalived. This means that the IPv6 link local address will only be present on the master. """ - device = ip_lib.IPDevice(interface_name, namespace=self.ns_name) + device = ip_lib.IPDevice(interface_name, namespace=self.ha_namespace) ipv6_lladdr = ip_lib.get_ipv6_lladdr(device.link.address) if self._should_delete_ipv6_lladdr(ipv6_lladdr): @@ -233,8 +244,7 @@ class HaRouter(router.RouterInfo): self._add_vip(ipv6_lladdr, interface_name, scope='link') def _add_gateway_vip(self, ex_gw_port, interface_name): - for ip_cidr in common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips']): - self._add_vip(ip_cidr, interface_name) + self._add_vips(ex_gw_port, interface_name) self._add_default_gw_virtual_route(ex_gw_port, interface_name) self._add_extra_subnet_onlink_routes(ex_gw_port, interface_name) @@ -254,20 +264,23 @@ class HaRouter(router.RouterInfo): for ip_cidr in ip_cidrs: self._add_vip(ip_cidr, interface_name) - def internal_network_added(self, port): + def _plug_ha_router_port(self, port, name_getter, prefix): port_id = port['id'] - interface_name = self.get_internal_device_name(port_id) + interface_name = name_getter(port_id) self.driver.plug(port['network_id'], port_id, interface_name, port['mac_address'], - namespace=self.ns_name, - prefix=router.INTERNAL_DEV_PREFIX) + namespace=self.ha_namespace, + prefix=prefix) self._disable_ipv6_addressing_on_interface(interface_name) - for ip_cidr in common_utils.fixed_ip_cidrs(port['fixed_ips']): - self._add_vip(ip_cidr, interface_name) + self._add_vips(port, interface_name) + + def internal_network_added(self, port): + self._plug_ha_router_port( + port, self.get_internal_device_name, router.INTERNAL_DEV_PREFIX) def internal_network_removed(self, port): super(HaRouter, self).internal_network_removed(port) @@ -279,7 +292,7 @@ class HaRouter(router.RouterInfo): return external_process.ProcessManager( self.agent_conf, '%s.monitor' % self.router_id, - self.ns_name, + self.ha_namespace, default_cmd_callback=self._get_state_change_monitor_callback()) def _get_state_change_monitor_callback(self): @@ -290,7 +303,7 @@ class HaRouter(router.RouterInfo): cmd = [ 'neutron-keepalived-state-change', '--router_id=%s' % self.router_id, - '--namespace=%s' % self.ns_name, + '--namespace=%s' % self.ha_namespace, '--conf_dir=%s' % self.keepalived_manager.get_conf_dir(), '--monitor_interface=%s' % ha_device, '--monitor_cidr=%s' % ha_cidr, @@ -317,7 +330,7 @@ class HaRouter(router.RouterInfo): def update_initial_state(self, callback): ha_device = ip_lib.IPDevice( self.get_ha_device_name(), - self.ns_name) + self.ha_namespace) addresses = ha_device.addr.list() cidrs = (address['cidr'] for address in addresses) ha_cidr = self._get_primary_vip() @@ -341,7 +354,8 @@ class HaRouter(router.RouterInfo): self._disable_ipv6_addressing_on_interface(interface_name) def external_gateway_updated(self, ex_gw_port, interface_name): - self._plug_external_gateway(ex_gw_port, interface_name, self.ns_name) + self._plug_external_gateway( + ex_gw_port, interface_name, self.ha_namespace) ip_cidrs = common_utils.fixed_ip_cidrs(self.ex_gw_port['fixed_ips']) for old_gateway_cidr in ip_cidrs: self._remove_vip(old_gateway_cidr) diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py index 84b16dfb362..02c83a39544 100644 --- a/neutron/tests/functional/agent/test_l3_agent.py +++ b/neutron/tests/functional/agent/test_l3_agent.py @@ -80,7 +80,7 @@ class L3AgentTestFramework(base.BaseSudoTestCase): agent_config.register_process_monitor_opts(config) return config - def _configure_agent(self, host): + def _configure_agent(self, host, agent_mode='dvr_snat'): conf = self._get_config_opts() l3_agent_main.register_opts(conf) conf.set_override( @@ -107,10 +107,14 @@ class L3AgentTestFramework(base.BaseSudoTestCase): conf.set_override('external_pids', get_temp_file_path('external/pids')) conf.set_override('host', host) + conf.set_override('agent_mode', agent_mode) agent = neutron_l3_agent.L3NATAgentWithStateReport(host, conf) return agent + def _get_agent_ovs_integration_bridge(self, agent): + return get_ovs_bridge(agent.conf.ovs_integration_bridge) + def generate_router_info(self, enable_ha, ip_version=4, extra_routes=True, enable_fip=True, enable_snat=True, dual_stack=False, v6_ext_gw_with_sub=True): @@ -286,11 +290,13 @@ class L3AgentTestFramework(base.BaseSudoTestCase): for extra_route in router.router['routes']: self.assertIn(extra_route, routes) - def _assert_onlink_subnet_routes(self, router, ip_versions): + def _assert_onlink_subnet_routes( + self, router, ip_versions, namespace=None): + ns_name = namespace or router.ns_name routes = [] for ip_version in ip_versions: _routes = ip_lib.get_routing_table(ip_version, - namespace=router.ns_name) + namespace=ns_name) routes.extend(_routes) routes = set(route['destination'] for route in routes) extra_subnets = router.get_ex_gw_port()['extra_subnets'] @@ -317,9 +323,23 @@ class L3AgentTestFramework(base.BaseSudoTestCase): def fail_ha_router(self, router): device_name = router.get_ha_device_name() - ha_device = ip_lib.IPDevice(device_name, router.ns_name) + ha_device = ip_lib.IPDevice(device_name, router.ha_namespace) ha_device.link.set_down() + @classmethod + def _get_addresses_on_device(cls, namespace, interface): + return [address['cidr'] for address in + ip_lib.IPDevice(interface, namespace=namespace).addr.list()] + + def _assert_no_ip_addresses_on_interface(self, namespace, interface): + self.assertEqual( + [], self._get_addresses_on_device(namespace, interface)) + + def _assert_ip_address_on_interface(self, + namespace, interface, ip_address): + self.assertIn( + ip_address, self._get_addresses_on_device(namespace, interface)) + class L3AgentTestCase(L3AgentTestFramework): @@ -615,8 +635,10 @@ class L3AgentTestCase(L3AgentTestFramework): slaac = l3_constants.IPV6_SLAAC slaac_mode = {'ra_mode': slaac, 'address_mode': slaac} subnet_modes = [slaac_mode] * 2 - self._add_internal_interface_by_subnet(router.router, count=2, - ip_version=6, ipv6_subnet_modes=subnet_modes) + self._add_internal_interface_by_subnet(router.router, + count=2, + ip_version=6, + ipv6_subnet_modes=subnet_modes) router.process(self.agent) if enable_ha: @@ -712,15 +734,6 @@ class L3AgentTestCase(L3AgentTestFramework): router.router[l3_constants.HA_INTERFACE_KEY], ha_router_dev_name_getter, router.ns_name)) - @classmethod - def _get_addresses_on_device(cls, namespace, interface): - return [address['cidr'] for address in - ip_lib.IPDevice(interface, namespace=namespace).addr.list()] - - def _assert_no_ip_addresses_on_interface(self, namespace, interface): - self.assertEqual( - [], self._get_addresses_on_device(namespace, interface)) - def test_ha_router_conf_on_restarted_agent(self): router_info = self.generate_router_info(enable_ha=True) router1 = self.manage_router(self.agent, router_info) @@ -777,9 +790,8 @@ class L3HATestFramework(L3AgentTestFramework): super(L3HATestFramework, self).setUp() self.failover_agent = self._configure_agent('agent2') - br_int_1 = get_ovs_bridge(self.agent.conf.ovs_integration_bridge) - br_int_2 = get_ovs_bridge( - self.failover_agent.conf.ovs_integration_bridge) + br_int_1 = self._get_agent_ovs_integration_bridge(self.agent) + br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent) veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports br_int_1.add_port(veth1.name) @@ -1013,6 +1025,9 @@ class TestDvrRouter(L3AgentTestFramework): def test_dvr_router_lifecycle_without_ha_with_snat_with_fips(self): self._dvr_router_lifecycle(enable_ha=False, enable_snat=True) + def test_dvr_router_lifecycle_ha_with_snat_with_fips(self): + self._dvr_router_lifecycle(enable_ha=True, enable_snat=True) + def _helper_create_dvr_router_fips_for_ext_network( self, agent_mode, **dvr_router_kwargs): self.agent.conf.agent_mode = agent_mode @@ -1054,7 +1069,9 @@ class TestDvrRouter(L3AgentTestFramework): self._validate_fips_for_external_network(router2, fip2_ns) def _dvr_router_lifecycle(self, enable_ha=False, enable_snat=False, - custom_mtu=2000): + custom_mtu=2000, + ip_version=4, + dual_stack=False): '''Test dvr router lifecycle :param enable_ha: sets the ha value for the router. @@ -1089,9 +1106,28 @@ class TestDvrRouter(L3AgentTestFramework): # manage the router (create it, create namespaces, # attach interfaces, etc...) router = self.manage_router(self.agent, router_info) + if enable_ha: + port = router.get_ex_gw_port() + interface_name = router.get_external_device_name(port['id']) + self._assert_no_ip_addresses_on_interface(router.ha_namespace, + interface_name) + utils.wait_until_true(lambda: router.ha_state == 'master') + + # Keepalived notifies of a state transition when it starts, + # not when it ends. Thus, we have to wait until keepalived finishes + # configuring everything. We verify this by waiting until the last + # device has an IP address. + device = router.router[l3_constants.INTERFACE_KEY][-1] + device_exists = functools.partial( + self.device_exists_with_ips_and_mac, + device, + router.get_internal_device_name, + router.ns_name) + utils.wait_until_true(device_exists) self.assertTrue(self._namespace_exists(router.ns_name)) - self.assertTrue(self._metadata_proxy_exists(self.agent.conf, router)) + utils.wait_until_true( + lambda: self._metadata_proxy_exists(self.agent.conf, router)) self._assert_internal_devices(router) self._assert_dvr_external_device(router) self._assert_dvr_gateway(router) @@ -1101,13 +1137,25 @@ class TestDvrRouter(L3AgentTestFramework): self._assert_metadata_chains(router) self._assert_extra_routes(router) self._assert_rfp_fpr_mtu(router, custom_mtu) + if enable_snat: + ip_versions = [4, 6] if (ip_version == 6 or dual_stack) else [4] + snat_ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name( + router.router_id) + self._assert_onlink_subnet_routes( + router, ip_versions, snat_ns_name) self._delete_router(self.agent, router.router_id) self._assert_interfaces_deleted_from_ovs() self._assert_router_does_not_exist(router) + self._assert_snat_namespace_does_not_exist(router) - def generate_dvr_router_info( - self, enable_ha=False, enable_snat=False, **kwargs): + def generate_dvr_router_info(self, + enable_ha=False, + enable_snat=False, + agent=None, + **kwargs): + if not agent: + agent = self.agent router = l3_test_common.prepare_router_data( enable_snat=enable_snat, enable_floating_ip=True, @@ -1115,11 +1163,11 @@ class TestDvrRouter(L3AgentTestFramework): **kwargs) internal_ports = router.get(l3_constants.INTERFACE_KEY, []) router['distributed'] = True - router['gw_port_host'] = self.agent.conf.host - router['gw_port']['binding:host_id'] = self.agent.conf.host + router['gw_port_host'] = agent.conf.host + router['gw_port']['binding:host_id'] = agent.conf.host floating_ip = router['_floatingips'][0] floating_ip['floating_network_id'] = router['gw_port']['network_id'] - floating_ip['host'] = self.agent.conf.host + floating_ip['host'] = agent.conf.host floating_ip['port_id'] = internal_ports[0]['id'] floating_ip['status'] = 'ACTIVE' @@ -1199,9 +1247,12 @@ class TestDvrRouter(L3AgentTestFramework): # that the correct ports and ip addresses exist in the # snat_ns_name namespace if self.agent.conf.agent_mode == 'dvr_snat': - self.assertTrue(self.device_exists_with_ips_and_mac( - external_port, router.get_external_device_name, - snat_ns_name)) + device_exists = functools.partial( + self.device_exists_with_ips_and_mac, + external_port, + router.get_external_device_name, + snat_ns_name) + utils.wait_until_true(device_exists) # if the agent is in dvr mode then the snat_ns_name namespace # should not be present at all: elif self.agent.conf.agent_mode == 'dvr': @@ -1484,3 +1535,104 @@ class TestDvrRouter(L3AgentTestFramework): self.agent.context, floating_agent_gw_port[0]['network_id']) self.assertFalse(self._namespace_exists(fip_ns)) + + def _mocked_dvr_ha_router(self, agent): + r_info = self.generate_dvr_router_info(enable_ha=True, + enable_snat=True, + agent=agent) + + r_snat_ns_name = namespaces.build_ns_name(dvr_snat_ns.SNAT_NS_PREFIX, + r_info['id']) + + mocked_r_snat_ns_name = r_snat_ns_name + '@' + agent.host + r_ns_name = namespaces.build_ns_name(namespaces.NS_PREFIX, + r_info['id']) + + mocked_r_ns_name = r_ns_name + '@' + agent.host + + return r_info, mocked_r_ns_name, mocked_r_snat_ns_name + + def _setup_dvr_ha_agents(self): + self.agent.conf.agent_mode = 'dvr_snat' + + self.failover_agent = self._configure_agent('agent2') + self.failover_agent.conf.agent_mode = 'dvr_snat' + + def _setup_dvr_ha_bridges(self): + br_int_1 = self._get_agent_ovs_integration_bridge(self.agent) + br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent) + + veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports + br_int_1.add_port(veth1.name) + br_int_2.add_port(veth2.name) + + def _create_dvr_ha_router(self, agent): + get_ns_name = mock.patch.object(namespaces.RouterNamespace, + '_get_ns_name').start() + get_snat_ns_name = mock.patch.object(dvr_snat_ns.SnatNamespace, + 'get_snat_ns_name').start() + (r_info, + mocked_r_ns_name, + mocked_r_snat_ns_name) = self._mocked_dvr_ha_router(agent) + get_ns_name.return_value = mocked_r_ns_name + get_snat_ns_name.return_value = mocked_r_snat_ns_name + router = self.manage_router(agent, r_info) + return router + + def _assert_ip_addresses_in_dvr_ha_snat_namespace(self, router): + namespace = router.ha_namespace + ex_gw_port = router.get_ex_gw_port() + snat_port = router.get_snat_interfaces()[0] + ex_gw_port_name = router.get_external_device_name( + ex_gw_port['id']) + snat_port_name = router._get_snat_int_device_name( + snat_port['id']) + + ip = ex_gw_port["fixed_ips"][0]['ip_address'] + prefix_len = ex_gw_port["fixed_ips"][0]['prefixlen'] + ex_gw_port_cidr = ip + "/" + str(prefix_len) + ip = snat_port["fixed_ips"][0]['ip_address'] + prefix_len = snat_port["fixed_ips"][0]['prefixlen'] + snat_port_cidr = ip + "/" + str(prefix_len) + + self._assert_ip_address_on_interface(namespace, + ex_gw_port_name, + ex_gw_port_cidr) + self._assert_ip_address_on_interface(namespace, + snat_port_name, + snat_port_cidr) + + def _assert_no_ip_addresses_in_dvr_ha_snat_namespace(self, router): + namespace = router.ha_namespace + ex_gw_port = router.get_ex_gw_port() + snat_port = router.get_snat_interfaces()[0] + ex_gw_port_name = router.get_external_device_name( + ex_gw_port['id']) + snat_port_name = router._get_snat_int_device_name( + snat_port['id']) + + self._assert_no_ip_addresses_on_interface(namespace, + snat_port_name) + self._assert_no_ip_addresses_on_interface(namespace, + ex_gw_port_name) + + def test_dvr_ha_router_failover(self): + self._setup_dvr_ha_agents() + self._setup_dvr_ha_bridges() + + router1 = self._create_dvr_ha_router(self.agent) + router2 = self._create_dvr_ha_router(self.failover_agent) + + utils.wait_until_true(lambda: router1.ha_state == 'master') + utils.wait_until_true(lambda: router2.ha_state == 'backup') + + self._assert_ip_addresses_in_dvr_ha_snat_namespace(router1) + self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router2) + + self.fail_ha_router(router1) + + utils.wait_until_true(lambda: router2.ha_state == 'master') + utils.wait_until_true(lambda: router1.ha_state == 'backup') + + self._assert_ip_addresses_in_dvr_ha_snat_namespace(router2) + self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router1)