Addressing L3 HA keepalived failures in functional tests

Current testing of Keepalived was not configuring the connectivity
between 2 agent namespaces.
Added setting up the veth pair.

Also, bridges external qg-<id> and internal qr-<id> were removed
from agent1 namespace and moved to agent2 namespace, because they had
the same name.
Added patching the qg and qr bridges name creation to be different for
functional tests.

Change-Id: I82b3218091da4feb39a9e820d0e54639ae27c97d
Closes-Bug: #1580648
(cherry picked from commit 8d3f216e24)

neutron/tests/functional/agent/l3/framework.py

@@ -28,6 +28,8 @@ import textwrap
 from neutron.agent.common import config as agent_config
 from neutron.agent.common import ovs_lib
 from neutron.agent.l3 import agent as neutron_l3_agent
+from neutron.agent.l3 import namespaces
+from neutron.agent.l3 import router_info as l3_router_info
 from neutron.agent import l3_agent as l3_agent_main
 from neutron.agent.linux import external_process
 from neutron.agent.linux import ip_lib
@@ -48,6 +50,7 @@ def get_ovs_bridge(br_name):
 
 class L3AgentTestFramework(base.BaseSudoTestCase):
     INTERFACE_DRIVER = 'neutron.agent.linux.interface.OVSInterfaceDriver'
+    NESTED_NAMESPACE_SEPARATOR = '@'
 
     def setUp(self):
         super(L3AgentTestFramework, self).setUp()
@@ -205,12 +208,15 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
             self._gateway_check(self.agent.conf.ipv6_gateway,
                                 external_device)
 
-    def _assert_external_device(self, router):
+    def _check_external_device(self, router):
         external_port = router.get_ex_gw_port()
-        self.assertTrue(self.device_exists_with_ips_and_mac(
+        return (self.device_exists_with_ips_and_mac(
             external_port, router.get_external_device_name,
             router.ns_name))
 
+    def _assert_external_device(self, router):
+        self.assertTrue(self._check_external_device(router))
+
     def _assert_ipv6_accept_ra(self, router):
         external_port = router.get_ex_gw_port()
         external_device_name = router.get_external_device_name(
@@ -501,6 +507,81 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
                 external_port['mac_address'],
                 namespace=router.ns_name) for fip in floating_ips)
 
+    def _create_router(self, router_info, agent):
+
+        ns_name = "%s%s%s" % (
+            'qrouter-' + router_info['id'],
+            self.NESTED_NAMESPACE_SEPARATOR, agent.host)
+        ext_name = "qg-%s-%s" % (agent.host, _uuid()[-4:])
+        int_name = "qr-%s-%s" % (agent.host, _uuid()[-4:])
+
+        get_ns_name = mock.patch.object(
+            namespaces.RouterNamespace, '_get_ns_name').start()
+        get_ns_name.return_value = ns_name
+        get_ext_name = mock.patch.object(l3_router_info.RouterInfo,
+            'get_external_device_name').start()
+        get_ext_name.return_value = ext_name
+        get_int_name = mock.patch.object(l3_router_info.RouterInfo,
+            'get_internal_device_name').start()
+        get_int_name.return_value = int_name
+
+        router = self.manage_router(agent, router_info)
+
+        router_ext_name = mock.patch.object(router,
+            'get_external_device_name').start()
+        router_ext_name.return_value = get_ext_name.return_value
+        router_int_name = mock.patch.object(router,
+            'get_internal_device_name').start()
+        router_int_name.return_value = get_int_name.return_value
+
+        return router
+
+    def create_ha_routers(self):
+        router_info = self.generate_router_info(enable_ha=True)
+        router1 = self._create_router(router_info, self.agent)
+        self._add_fip(router1, '192.168.111.12')
+
+        r1_br = ip_lib.IPDevice(router1.driver.conf.external_network_bridge)
+        r1_br.addr.add('19.4.4.1/24')
+        r1_br.link.set_up()
+
+        router_info_2 = copy.deepcopy(router_info)
+        router_info_2[constants.HA_INTERFACE_KEY] = (
+            l3_test_common.get_ha_interface(ip='169.254.192.2',
+                                            mac='22:22:22:22:22:22'))
+        router2 = self._create_router(router_info_2, self.failover_agent)
+
+        r2_br = ip_lib.IPDevice(router2.driver.conf.external_network_bridge)
+        r2_br.addr.add('19.4.4.1/24')
+        r2_br.link.set_up()
+
+        return (router1, router2)
+
+    def _get_master_and_slave_routers(self, router1, router2):
+
+        try:
+            common_utils.wait_until_true(
+                lambda: router1.ha_state == 'master')
+            common_utils.wait_until_true(
+                lambda: self._check_external_device(router1))
+            master_router = router1
+            slave_router = router2
+        except common_utils.WaitTimeout:
+            common_utils.wait_until_true(
+                lambda: router2.ha_state == 'master')
+            common_utils.wait_until_true(
+                lambda: self._check_external_device(router2))
+            master_router = router2
+            slave_router = router1
+
+        common_utils.wait_until_true(
+                lambda: master_router.ha_state == 'master')
+        common_utils.wait_until_true(
+                lambda: self._check_external_device(master_router))
+        common_utils.wait_until_true(
+            lambda: slave_router.ha_state == 'backup')
+        return master_router, slave_router
+
     def fail_ha_router(self, router):
         device_name = router.get_ha_device_name()
         ha_device = ip_lib.IPDevice(device_name, router.ha_namespace)

neutron/tests/functional/agent/l3/test_ha_router.py

@@ -21,7 +21,6 @@ import six
 import testtools
 
 from neutron.agent.l3 import agent as neutron_l3_agent
-from neutron.agent.l3 import namespaces
 from neutron.agent.linux import ip_lib
 from neutron.common import ipv6_utils
 from neutron.common import utils as common_utils
@@ -305,8 +304,6 @@ class L3HATestCase(framework.L3AgentTestFramework):
 
 class L3HATestFailover(framework.L3AgentTestFramework):
 
-    NESTED_NAMESPACE_SEPARATOR = '@'
-
     def setUp(self):
         super(L3HATestFailover, self).setUp()
         conf = self._configure_agent('agent2')
@@ -317,35 +314,26 @@ class L3HATestFailover(framework.L3AgentTestFramework):
         br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent)
 
         veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports
+        veth1.link.set_up()
+        veth2.link.set_up()
         br_int_1.add_port(veth1.name)
         br_int_2.add_port(veth2.name)
 
     def test_ha_router_failover(self):
-        router_info = self.generate_router_info(enable_ha=True)
+        router1, router2 = self.create_ha_routers()
-        get_ns_name = mock.patch.object(
-            namespaces.RouterNamespace, '_get_ns_name').start()
-        get_ns_name.return_value = "%s%s%s" % (
-            'qrouter-' + router_info['id'],
-            self.NESTED_NAMESPACE_SEPARATOR, self.agent.host)
-        router1 = self.manage_router(self.agent, router_info)
 
-        router_info_2 = copy.deepcopy(router_info)
+        master_router, slave_router = self._get_master_and_slave_routers(
-        router_info_2[constants.HA_INTERFACE_KEY] = (
+            router1, router2)
-            l3_test_common.get_ha_interface(ip='169.254.192.2',
-                                            mac='22:22:22:22:22:22'))
-
-        get_ns_name.return_value = "%s%s%s" % (
-            namespaces.RouterNamespace._get_ns_name(router_info_2['id']),
-            self.NESTED_NAMESPACE_SEPARATOR, self.failover_agent.host)
-        router2 = self.manage_router(self.failover_agent, router_info_2)
-
-        common_utils.wait_until_true(lambda: router1.ha_state == 'master')
-        common_utils.wait_until_true(lambda: router2.ha_state == 'backup')
 
         self.fail_ha_router(router1)
 
-        common_utils.wait_until_true(lambda: router2.ha_state == 'master')
+        # NOTE: passing slave_router as first argument, because we expect
-        common_utils.wait_until_true(lambda: router1.ha_state == 'backup')
+        # that this router should be the master
+        new_master, new_slave = self._get_master_and_slave_routers(
+            slave_router, master_router)
+
+        self.assertEqual(master_router, new_slave)
+        self.assertEqual(slave_router, new_master)
 
 
 class LinuxBridgeL3HATestCase(L3HATestCase):