Merge "Add Active Active L3 GW scenario test cases"

doc/source/conf.py

@@ -16,7 +16,10 @@ import os
 import sys
 
 def autodoc_skip_member_handler(app, what, name, obj, skip, options):
-    return skip or (what == "class" and not name.startswith("test"))
+    return skip or (
+        (what == "class" and not name.startswith("test")) or
+        # NOTE(fnordahl): Sphinx does not like the ASCII art in the docstring.
+        (what == 'module' and name == 'NetworkMultipleGWTest'))
 
 def setup(app):
     app.connect('autodoc-skip-member', autodoc_skip_member_handler)

neutron_tempest_plugin/api/base.py

@@ -875,9 +875,10 @@ class BaseNetworkTest(test.BaseTestCase):
                                            association['fixed_port_id'])
 
     @classmethod
-    def create_router_interface(cls, router_id, subnet_id):
+    def create_router_interface(cls, router_id, subnet_id, client=None):
         """Wrapper utility that returns a router interface."""
-        interface = cls.client.add_router_interface_with_subnet_id(
+        client = client or cls.client
+        interface = client.add_router_interface_with_subnet_id(
             router_id, subnet_id)
         return interface
 
@@ -1350,13 +1351,14 @@ class BaseAdminNetworkTest(BaseNetworkTest):
 
     @classmethod
     def create_provider_network(cls, physnet_name, start_segmentation_id,
-                                max_attempts=30):
+                                max_attempts=30, external=False):
         segmentation_id = start_segmentation_id
         for attempts in range(max_attempts):
             try:
                 return cls.create_network(
                     name=data_utils.rand_name('test_net'),
-                    shared=True,
+                    shared=not external,
+                    external=external,
                     provider_network_type='vlan',
                     provider_physical_network=physnet_name,
                     provider_segmentation_id=segmentation_id)

neutron_tempest_plugin/api/test_routers.py

@@ -329,13 +329,10 @@ class RoutersTest(base_routers.BaseRouterTest):
 class ExternalGWMultihomingRoutersTest(base_routers.BaseRouterTest):
 
     @classmethod
+    @tutils.requires_ext(extension="external-gateway-multihoming",
+                         service="network")
     def setUpClass(cls):
         super().setUpClass()
-        ext_alias = 'external-gateway-multihoming'
-        try:
-            cls.client.get_extension(ext_alias)
-        except lib_exc.NotFound:
-            raise cls.skipException(f'{ext_alias} extension not available.')
 
     @decorators.idempotent_id('33e9a156-a83f-435f-90ee-1a49dc9c350d')
     def test_create_router_enable_default_route_ecmp(self):

neutron_tempest_plugin/scenario/base.py

@@ -286,6 +286,7 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
 
     def setup_network_and_server(self, router=None, server_name=None,
                                  network=None, use_stateless_sg=False,
+                                 create_fip=True, router_client=None,
                                  **kwargs):
         """Create network resources and a server.
 
@@ -309,7 +310,8 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         self.security_groups.append(secgroup['security_group'])
         if not router:
             router = self.create_router_by_client(**kwargs)
-        self.create_router_interface(router['id'], self.subnet['id'])
+        self.create_router_interface(router['id'], self.subnet['id'],
+                                     client=router_client)
         self.keypair = self.create_keypair()
         self.create_loginable_secgroup_rule(
             secgroup_id=secgroup['security_group']['id'])
@@ -331,7 +333,9 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         self.port = self.client.list_ports(network_id=self.network['id'],
                                            device_id=self.server[
                                                'server']['id'])['ports'][0]
-        self.fip = self.create_floatingip(port=self.port)
+
+        if create_fip:
+            self.fip = self.create_floatingip(port=self.port)
 
     def check_connectivity(self, host, ssh_user=None, ssh_key=None,
                            servers=None, ssh_timeout=None, ssh_client=None):
@@ -696,3 +700,8 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         except exceptions.SSHScriptFailed:
             raise self.skipException(
                 "%s is not available on server %s" % (cmd, server['id']))
+
+
+class BaseAdminTempestTestCase(base_api.BaseAdminNetworkTest,
+                               BaseTempestTestCase):
+    pass

neutron_tempest_plugin/scenario/test_multiple_gws.py

@@ -0,0 +1,750 @@
+# Copyright 2023 Canonical
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import json
+import os
+import subprocess
+import time
+import typing
+
+import netaddr
+import testtools
+
+from tempest.common import utils as tutils
+
+from neutron_tempest_plugin import config
+from neutron_tempest_plugin.scenario import base
+
+from neutron_lib import constants as const
+
+from oslo_log import log
+
+from os_ken.tests.integrated.common import docker_base as ctn_base
+
+from tempest.lib.common.utils import data_utils
+from tempest.lib import decorators
+from tempest.lib import exceptions as lib_exc
+
+CONF = config.CONF
+LOG = log.getLogger(__name__)
+
+
+class FRROCIImage(ctn_base.DockerImage):
+    def __init__(
+        self,
+        daemons: typing.Tuple[str],
+        baseimage: typing.Optional[str] = None,
+        use_existing: bool = False,
+    ):
+        super().__init__(baseimage=baseimage or 'ubuntu:22.04')
+        self.daemons = daemons
+        self.tagname = 'frr-' + '-'.join(daemons)
+        if use_existing and self.exist(self.tagname):
+            return
+
+        workdir = os.path.join(ctn_base.TEST_BASE_DIR, self.tagname)
+        pkgs = ' '.join(('telnet', 'tcpdump', 'frr'))
+        c = ctn_base.CmdBuffer()
+        c << f'FROM {self.baseimage}'
+        c << 'RUN apt-get update'
+        c << f'RUN apt-get install -qy --no-install-recommends {pkgs}'
+        c << 'RUN echo "#!/bin/sh" > /frr'
+        c << 'RUN echo mkdir -p /run/frr >> /frr'
+        c << 'RUN echo chmod 755 /run/frr >> /frr'
+        c << 'RUN echo chown frr:frr /run/frr >> /frr'
+        c << (
+            'RUN echo exec /usr/lib/frr/watchfrr '
+            f'-F traditional {" ".join(self.daemons)}>> /frr'
+        )
+        c << 'RUN chmod +x /frr'
+        c << 'CMD /frr'
+
+        self.cmd.sudo(f'rm -rf {workdir}')
+        self.cmd.execute(f'mkdir -p {workdir}')
+        self.cmd.execute(f"echo '{str(c)}' > {workdir}/Dockerfile")
+        self.build(self.tagname, workdir)
+
+
+class FRRContainer(ctn_base.Container):
+    class veth_info(typing.NamedTuple):
+        bridge_name: str
+        bridge_type: str
+        ctn_ifname: str
+        host_ifname: str
+
+    _veths: typing.List[veth_info]
+
+    class route(typing.NamedTuple):
+        dst: netaddr.IPNetwork
+        next_hop: netaddr.IPNetwork
+
+    _ctn_routes: typing.List[route]
+
+    def __init__(
+        self,
+        name: str,
+        image: FRROCIImage,
+    ):
+        self._veths = []
+        self._ctn_routes = []
+        super().__init__(name, image.tagname)
+
+    # XXX upstream to os-ken
+    def next_if_name(self) -> str:
+        name = 'eth{0}'.format(len(self.eths))
+        self.eths.append(name)
+        return name
+
+    # XXX upstream to os-ken
+    def run(self, network: typing.Optional[str] = None) -> int:
+        c = ctn_base.CmdBuffer(' ')
+        c << "docker run --privileged=true"
+        for sv in self.shared_volumes:
+            c << "-v {0}:{1}".format(sv[0], sv[1])
+        if network:
+            c << "--network {0}".format(network)
+        c << "--name {0} --hostname {0} -id {1}".format(
+            self.docker_name(), self.image
+        )
+        self.id = self.dcexec(str(c), retry=True)
+        self.is_running = True
+        self.exec_on_ctn("ip li set up dev lo")
+        ipv4 = None
+        ipv6 = None
+        if network and network != 'none':
+            ifname = self.next_if_name()
+            for line in self.exec_on_ctn(f"ip a show dev {ifname}").split(
+                '\n'
+            ):
+                if line.strip().startswith("inet "):
+                    elems = [e.strip() for e in line.strip().split(' ')]
+                    ipv4 = elems[1]
+                elif line.strip().startswith("inet6 "):
+                    elems = [e.strip() for e in line.strip().split(' ')]
+                    ipv6 = elems[1]
+            self.set_addr_info(
+                bridge='docker0', ipv4=ipv4, ipv6=ipv6, ifname=ifname
+            )
+        return 0
+
+    def wait_for_frr_daemons_up(
+        self,
+        try_times: int = 30,
+        interval: int = 1,
+    ) -> ctn_base.CommandOut:
+        return self.cmd.sudo(
+            f'docker logs {self.docker_name()} '
+            '|grep "WATCHFRR.*all daemons up"',
+            try_times=try_times,
+            interval=interval,
+        )
+
+    @staticmethod
+    def hash_ifname(ifname: str) -> str:
+        # Assuming IFNAMSIZ of 16, with null-termination gives 15 characters.
+        return 'veth' + str(hash(ifname) % 10**11)
+
+    @staticmethod
+    def get_if_mac(ifname: str) -> netaddr.EUI:
+        with open(f'/sys/class/net/{ifname}/address') as faddr:
+            return faddr.readline().rstrip()
+
+    def add_veth_to_bridge(
+        self,
+        bridge_name: str,
+        bridge_type: str,
+        ipv4_cidr: str,
+        ipv6_cidr: str,
+        ipv6_prefix: typing.Optional[netaddr.IPNetwork] = None,
+        vlan: typing.Optional[int] = None,
+    ) -> None:
+        assert self.is_running, (
+            'the container must be running before '
+            'calling add_veth_to_bridge'
+        )
+        assert (
+            bridge_type == ctn_base.BRIDGE_TYPE_OVS
+        ), f'bridge_type must be {ctn_base.BRIDGE_TYPE_OVS}'
+        veth_pair = (
+            self.hash_ifname(f'{self.name}-int{len(self._veths)}'),
+            self.hash_ifname(f'{self.name}-ext{len(self._veths)}'),
+        )
+        self.cmd.sudo(
+            f'ip link add {veth_pair[0]} type veth peer name {veth_pair[1]}'
+        )
+        if ipv6_prefix and not ipv6_cidr:
+            eui = netaddr.EUI(self.get_if_mac(veth_pair[0]))
+            ipv6_cidr = (
+                f'{eui.ipv6(ipv6_prefix.first)}/{ipv6_prefix.prefixlen}'
+            )
+
+        self.cmd.sudo(f'ip link set netns {self.get_pid()} dev {veth_pair[0]}')
+        self.cmd.sudo(f'ovs-vsctl add-port {bridge_name} {veth_pair[1]}')
+        if vlan:
+            self.cmd.sudo(f'ovs-vsctl set port {veth_pair[1]} tag={vlan}')
+
+        ifname = self.next_if_name()
+        self.exec_on_ctn(f'ip link set name {ifname} {veth_pair[0]}')
+
+        # Ensure IPv6 is not disabled in container
+        self.exec_on_ctn('sysctl -w net.ipv6.conf.all.disable_ipv6=0')
+
+        for cidr in (ipv4_cidr, ipv6_cidr):
+            if not cidr:
+                continue
+            self.exec_on_ctn(f'ip addr add {cidr} dev {ifname}')
+        self.exec_on_ctn(f'ip link set up dev {ifname}')
+        self.cmd.sudo(f'ip link set up dev {veth_pair[1]}')
+        self.set_addr_info(
+            bridge_name, ipv4=ipv4_cidr, ipv6=ipv6_cidr, ifname=ifname
+        )
+        self._veths.append(
+            self.veth_info(
+                bridge_name=bridge_name,
+                bridge_type=bridge_type,
+                ctn_ifname=ifname,
+                host_ifname=veth_pair[1],
+            )
+        )
+
+    def add_ctn_route(self, route: route) -> None:
+        self.exec_on_ctn(
+            f'ip -{route.dst.version} route add '
+            f'{str(route.dst.cidr)} via {str(route.next_hop.ip)}'
+        )
+        self._ctn_routes.append(route)
+
+    def del_ctn_route(self, route: route) -> None:
+        self.exec_on_ctn(
+            f'ip -{route.dst.version} route del '
+            f'{str(route.dst.cidr)} via {str(route.next_hop.ip)}'
+        )
+        self._ctn_routes.remove(route)
+
+    def remove(self, check_exist=True) -> ctn_base.CommandOut:
+        for veth in self._veths:
+            # The veth pair itself will be destroyed as a side effect of
+            # removing the container, so we only need to clean up the bridge
+            # attachment.
+            if veth.bridge_type == ctn_base.BRIDGE_TYPE_BRCTL:
+                self.cmd.sudo(
+                    'brctl delif ' f'{veth.bridge_name} ' f'{veth.host_ifname}'
+                )
+            elif veth.bridge_type == ctn_base.BRIDGE_TYPE_OVS:
+                self.cmd.sudo(
+                    'ovs-vsctl del-port '
+                    f'{veth.bridge_name} '
+                    f'{veth.host_ifname}'
+                )
+        super().remove(check_exist=check_exist)
+
+    def vtysh(self, cmd: typing.List[str]) -> ctn_base.CommandOut:
+        cmd_str = ' '.join(f"-c '{c}'" for c in cmd)
+        return self.exec_on_ctn(f'vtysh {cmd_str}', capture=True)
+
+
+class BFDContainer(FRRContainer):
+    def __init__(
+        self,
+        name: str,
+        image: typing.Optional[FRROCIImage] = None,
+    ):
+        image = image or FRROCIImage(
+            daemons=('zebra', 'bfdd'), use_existing=True
+        )
+        super().__init__(name, image)
+        assert 'bfdd' in image.daemons
+
+    def add_bfd_peer(self, ip_address: str) -> None:
+        self.vtysh(
+            [
+                'enable',
+                'conf',
+                'bfd',
+                f'peer {ip_address} interface eth0',
+            ]
+        )
+
+    def del_bfd_peer(self, ip_address: str) -> None:
+        self.vtysh(
+            [
+                'enable',
+                'conf',
+                'bfd',
+                f'no peer {ip_address} interface eth0',
+            ]
+        )
+
+    def show_bfd_peer(self, peer: str) -> typing.Dict[str, typing.Any]:
+        return json.loads(self.vtysh([f'show bfd peer {peer} json']))
+
+    def wait_for_bfd_peer_status(
+        self, peer: str, status: str, try_times=30, interval=1
+    ) -> None:
+        while try_times:
+            peer_data = self.show_bfd_peer(peer)
+            if peer_data['status'] == status:
+                return
+            time.sleep(interval)
+            try_times -= 1
+        raise lib_exc.TimeoutException
+
+
+class NetworkMultipleGWTest(base.BaseAdminTempestTestCase):
+    """Test the following topology
+
+    +------------------------------------------------------------------+
+    |                          test runner                             |
+    |                                                                  |
+    |                                 +-----------+ eth0 public VLAN N |
+    | +-------- br-ex ----------+     | FRR w/BFD |                    |
+    | | +---------------------+ |     +-----------+ eth1 public flat   |
+    | | |   public physnet    | |     +-----------+ eth0 public VLAN N |
+    | | +---------------------+ |     | FRR w/BFD |                    |
+    | +-------------------------+     +-----------+ eth1 public flat   |
+    |     |              |                                             |
+    +-----|--------------|---------------------------------------------+
+          | -  VLAN N  - |
+     +-------------------------+
+     |      project router     | - enable_default_route_{bfd,ecmp}=True
+     +-------------------------+
+                 |
+           +----------+
+           | instance |
+           +----------+
+
+    NOTE(fnordahl) At the time of writing, FRR provides a BFD daemon, but has
+    not integrated it with static routes [0][1].  As a consequence the
+    test will manually add/remove routes on test runner to ensure correct path
+    is chosen for traffic from test runner to instance.  On the return path the
+    BFD implementation in OVN will ensure the correct path is chosen
+    automatically.
+
+    In real world usage most vendors have BFD support for static routes.
+
+    0: https://github.com/FRRouting/frr/wiki/Feature-Requests
+    1: https://github.com/FRRouting/frr/issues/3369
+    """
+    class host_route(typing.NamedTuple):
+        dst: netaddr.IPNetwork
+        next_hop: netaddr.IPNetwork
+
+    host_routes: typing.List[host_route] = []
+
+    credentials = ['primary', 'admin']
+
+    @classmethod
+    def setup_clients(cls):
+        super().setup_clients()
+        if not cls.admin_client:
+            cls.admin_client = cls.os_admin.network_client
+
+    @classmethod
+    @tutils.requires_ext(extension="external-gateway-multihoming",
+                         service="network")
+    def resource_setup(cls):
+        super().resource_setup()
+
+        # Ensure devstack configured public subnets are recorded, so that we
+        # don't attempt to use them again.
+        cls.reserve_external_subnet_cidrs()
+
+        # We need to know prefixlength of the devstack configured public
+        # subnets.
+        for subnet_id in cls.admin_client.show_network(
+                CONF.network.public_network_id)['network']['subnets']:
+            subnet = cls.admin_client.show_subnet(subnet_id)['subnet']
+            if subnet['ip_version'] == 4:
+                cls.public_ipv4_subnet = subnet
+                continue
+            cls.public_ipv6_subnet = subnet
+        cls.ext_networks = []
+        for n in range(0, 2):
+            ext_network = cls.create_provider_network(
+                physnet_name='public',
+                start_segmentation_id=4040 + n,
+                external=True,
+            )
+            ext_ipv6_subnet = cls.create_subnet(
+                ext_network,
+                ip_version=const.IP_VERSION_6,
+                client=cls.admin_client,
+            )
+            ext_ipv4_subnet = cls.create_subnet(
+                ext_network,
+                ip_version=const.IP_VERSION_4,
+                client=cls.admin_client,
+            )
+            cls.ext_networks.append(
+                (ext_network, ext_ipv6_subnet, ext_ipv4_subnet)
+            )
+        cls.host_routes = []
+        cls.resource_setup_container()
+
+    @classmethod
+    def resource_setup_container(cls):
+        cls.containers = []
+        for n in range(0, 2):
+            ext_network, ext_ipv6_subnet, ext_ipv4_subnet = cls.ext_networks[n]
+
+            # frr container
+            bfd_container = BFDContainer(data_utils.rand_name('frr'))
+            cls.containers.append(bfd_container)
+
+            bfd_container.run(network='none')
+            public_ipv6_net = netaddr.IPNetwork(cls.public_ipv6_subnet['cidr'])
+            public_ipv4_net = netaddr.IPNetwork(cls.public_ipv4_subnet['cidr'])
+            ipv6_net = netaddr.IPNetwork(ext_ipv6_subnet['cidr'])
+            ipv4_net = netaddr.IPNetwork(ext_ipv4_subnet['cidr'])
+            # reserve an IP for container on the public network for routing
+            # into the vlan network.
+            fip_address = cls.create_floatingip()['floating_ip_address']
+            cls.veths = [
+                bfd_container.add_veth_to_bridge(
+                    'br-ex',
+                    ctn_base.BRIDGE_TYPE_OVS,
+                    f'{ext_ipv4_subnet["gateway_ip"]}/{ipv4_net.prefixlen}',
+                    f'{ext_ipv6_subnet["gateway_ip"]}/{ipv6_net.prefixlen}',
+                    vlan=ext_network['provider:segmentation_id'],
+                ),
+                bfd_container.add_veth_to_bridge(
+                    'br-ex',
+                    ctn_base.BRIDGE_TYPE_OVS,
+                    f'{fip_address}/{public_ipv4_net.prefixlen}',
+                    '',
+                    ipv6_prefix=public_ipv6_net,
+                ),
+            ]
+            for subnet in (cls.public_ipv4_subnet, cls.public_ipv6_subnet):
+                bfd_container.exec_on_ctn(
+                    f'ip -{subnet["ip_version"]} route add default '
+                    f'via {subnet["gateway_ip"]} dev eth1'
+                )
+            for ip_version in (6, 4):
+                for addr_info in bfd_container.get_addr_info(
+                    'br-ex', ip_version
+                ).items():
+                    if addr_info[1] == 'eth1':
+                        if ip_version == 6:
+                            dst_subnet = ext_ipv6_subnet
+                        else:
+                            dst_subnet = ext_ipv4_subnet
+                        cls.add_host_route(
+                            cls.host_routes,
+                            cls.host_route(
+                                netaddr.IPNetwork(dst_subnet["cidr"]),
+                                netaddr.IPNetwork(addr_info[0]),
+                            ),
+                        )
+            bfd_container.wait_for_frr_daemons_up()
+
+    @classmethod
+    def resource_cleanup(cls):
+        # Ensure common cleanup code can clean up resources created by admin
+        cls.client = cls.admin_client
+        super().resource_cleanup()
+        for ctn in cls.containers:
+            try:
+                ctn.stop()
+            except ctn_base.CommandError:
+                pass
+            ctn.remove()
+        # NOTE(fnordahl): the loop body modifies the list, so we need to
+        # iterate on a copy.
+        for route in cls.host_routes.copy():
+            cls.del_host_route(cls.host_routes, route)
+
+    @staticmethod
+    def add_host_route(
+        lst: typing.List[host_route],
+        route: host_route
+    ) -> None:
+        subprocess.run(
+            (
+                'sudo',
+                'ip',
+                f'-{route.dst.version}',
+                'route',
+                'add',
+                str(route.dst.cidr),
+                'via',
+                str(route.next_hop.ip),
+            ),
+            capture_output=True,
+            check=True,
+            universal_newlines=True,
+        )
+        lst.append(route)
+
+    @staticmethod
+    def del_host_route(
+        lst: typing.List[host_route],
+        route: host_route
+    ) -> None:
+        subprocess.run(
+            (
+                'sudo',
+                'ip',
+                f'-{route.dst.version}',
+                'route',
+                'del',
+                str(route.dst.cidr),
+                'via',
+                str(route.next_hop.ip),
+            ),
+            capture_output=True,
+            check=True,
+            universal_newlines=True,
+        )
+        lst.remove(route)
+
+    def add_ctn_route(
+        self,
+        ctn: BFDContainer,
+        dst: netaddr.IPNetwork,
+        next_hop: netaddr.IPNetwork,
+    ):
+        ctn_route = ctn.route(dst, next_hop)
+        ctn.add_ctn_route(ctn_route)
+        self.per_test_ctn_routes.append((ctn, ctn_route))
+
+    def setUp(self):
+        super().setUp()
+        self.per_test_host_routes = []
+        self.per_test_ctn_routes = []
+
+    def tearDown(self):
+        super().tearDown()
+        # NOTE(fnordahl): the loop body modifies the list, so we need to
+        # iterate on a copy.
+        for ctn_route in self.per_test_ctn_routes.copy():
+            ctn = ctn_route[0]
+            route = ctn_route[1]
+            ctn.del_ctn_route(route)
+        for host_route in self.per_test_host_routes.copy():
+            self.del_host_route(self.per_test_host_routes, host_route)
+
+    def add_routes_for_router(
+        self,
+        router: typing.Dict[str, typing.Any],
+        ctn: FRRContainer,
+        add_ctn_route: bool = True,
+        add_host_route: bool = True,
+    ):
+        for port in self.admin_client.list_router_interfaces(router['id'])[
+            'ports'
+        ]:
+            if port['device_owner'] != const.DEVICE_OWNER_ROUTER_INTF:
+                continue
+            for fixed_ip in port['fixed_ips']:
+                subnet = self.client.show_subnet(
+                    fixed_ip['subnet_id'])['subnet']
+                for addr_info in ctn.get_addr_info(
+                    'br-ex',
+                    subnet['ip_version'],
+                ).items():
+                    if addr_info[1] == 'eth0':
+                        # container route
+                        ctn_net = netaddr.IPNetwork(addr_info[0])
+                        for gw_info in router['external_gateways']:
+                            for ip_info in gw_info['external_fixed_ips']:
+                                if (
+                                    ip_info['ip_address'] in ctn_net and
+                                    add_ctn_route
+                                ):
+                                    self.add_ctn_route(
+                                        ctn,
+                                        netaddr.IPNetwork(subnet['cidr']),
+                                        netaddr.IPNetwork(
+                                            ip_info['ip_address']
+                                        ),
+                                    )
+                    elif addr_info[1] == 'eth1' and add_host_route:
+                        self.add_host_route(
+                            self.per_test_host_routes,
+                            self.host_route(
+                                netaddr.IPNetwork(self.subnet['cidr']),
+                                netaddr.IPNetwork(addr_info[0]),
+                            ),
+                        )
+
+    @testtools.skipUnless(
+        CONF.compute.min_compute_nodes == 1,
+        'More than 1 compute node, test only works on '
+        'single node configurations.',
+    )
+    @decorators.idempotent_id('9baa05e6-ba10-4850-93e3-695f4d97b8f8')
+    def test_create_router_single_gw_bfd(self):
+        ext_network_id = self.ext_networks[0][0]['id']
+        bfd_container = self.containers[0]
+        router = self.create_admin_router(
+            router_name=data_utils.rand_name('router'),
+            admin_state_up=True,
+            enable_snat=False,
+            enable_default_route_bfd=True,
+            external_network_id=ext_network_id,
+        )
+        self.assertTrue(router['enable_default_route_bfd'])
+
+        # Add BFD peers on bfd_container.
+        for gw_info in router['external_gateways']:
+            for ip_info in gw_info['external_fixed_ips']:
+                bfd_container.add_bfd_peer(ip_info["ip_address"])
+                bfd_container.wait_for_bfd_peer_status(
+                    ip_info['ip_address'], 'up'
+                )
+
+        self.setup_network_and_server(
+            router=router,
+            create_fip=False,
+            router_client=self.admin_client,
+        )
+
+        self.add_routes_for_router(router, bfd_container)
+
+        # check connectivity
+        self.check_connectivity(
+            self.port['fixed_ips'][0]['ip_address'],
+            CONF.validation.image_ssh_user,
+            self.keypair['private_key'],
+        )
+
+    @testtools.skipUnless(
+        CONF.compute.min_compute_nodes == 1,
+        'More than 1 compute node, test only works on '
+        'single node configurations.',
+    )
+    @decorators.idempotent_id('75202251-c384-4962-8685-60cf2c530906')
+    def test_update_router_single_gw_bfd(self):
+        ext_network_id = self.ext_networks[0][0]['id']
+        bfd_container = self.containers[0]
+        router = self.create_router(
+            router_name=data_utils.rand_name('router'),
+            admin_state_up=True,
+            enable_snat=False,
+            external_network_id=ext_network_id,
+        )
+        self.assertFalse(router['enable_default_route_bfd'])
+
+        self.setup_network_and_server(
+            router=router,
+            create_fip=False,
+            router_client=self.admin_client,
+        )
+
+        self.add_routes_for_router(router, bfd_container)
+
+        # check connectivity
+        self.check_connectivity(
+            self.port['fixed_ips'][0]['ip_address'],
+            CONF.validation.image_ssh_user,
+            self.keypair['private_key'],
+        )
+
+        # Enable BFD on router.
+        #
+        # NOTE(fnordahl): We need to repeat the `enable_snat` state, otherwise
+        # the state will be toggled to the default value of 'True'.
+        router = self.admin_client.update_router_with_snat_gw_info(
+            router['id'],
+            enable_snat=False,
+            enable_default_route_bfd=True,
+        )['router']
+        self.assertTrue(router['enable_default_route_bfd'])
+
+        # Add BFD peers on bfd_container.
+        for gw_info in router['external_gateways']:
+            for ip_info in gw_info['external_fixed_ips']:
+                bfd_container.add_bfd_peer(ip_info["ip_address"])
+                bfd_container.wait_for_bfd_peer_status(
+                    ip_info['ip_address'], 'up'
+                )
+
+        # check connectivity
+        self.check_connectivity(
+            self.port['fixed_ips'][0]['ip_address'],
+            CONF.validation.image_ssh_user,
+            self.keypair['private_key'],
+        )
+
+    @testtools.skipUnless(
+        CONF.compute.min_compute_nodes == 1,
+        'More than 1 compute node, test only works on '
+        'single node configurations.',
+    )
+    @decorators.idempotent_id('5117587d-9633-48b7-aa8f-ec9d59a601a5')
+    def test_create_router_multiple_gw_bfd_and_ecmp(self):
+        router = self.create_admin_router(
+            router_name=data_utils.rand_name('router'),
+            admin_state_up=True,
+            enable_default_route_bfd=True,
+            enable_default_route_ecmp=True,
+        )
+        router = self.admin_client.router_add_external_gateways(
+            router['id'],
+            [
+                {
+                    'network_id': self.ext_networks[0][0]['id'],
+                    'enable_snat': False,
+                },
+                {
+                    'network_id': self.ext_networks[1][0]['id'],
+                    'enable_snat': False,
+                },
+            ],
+        )['router']
+
+        self.setup_network_and_server(
+            router=router,
+            create_fip=False,
+            router_client=self.admin_client,
+        )
+
+        # Add BFD peers on bfd_containers.
+        for gw_info in router['external_gateways']:
+            for ip_info in gw_info['external_fixed_ips']:
+                ip = netaddr.IPAddress(ip_info['ip_address'])
+                for ctn in self.containers:
+                    for addr_info in ctn.get_addr_info(
+                        'br-ex',
+                        ip.version,
+                    ).items():
+                        if addr_info[1] == 'eth0':
+                            ctn_net = netaddr.IPNetwork(addr_info[0])
+                            if ip not in ctn_net:
+                                break
+                            ctn.add_bfd_peer(str(ip))
+                            ctn.wait_for_bfd_peer_status(str(ip), 'up')
+
+        # Add route to project network on all containers.
+        for ctn in self.containers:
+            self.add_routes_for_router(router, ctn, True, False)
+
+        # Add host route to project network via FRR container and confirm
+        # connectivity one by one.
+        #
+        # We deliberately don't add both host routes at once as that would be
+        # testing test runner configuration and linux kernel ECMP, which is out
+        # of scope for our test.
+        for ctn in self.containers:
+            self.add_routes_for_router(router, ctn, False, True)
+
+            # check connectivity
+            self.check_connectivity(
+                self.port['fixed_ips'][0]['ip_address'],
+                CONF.validation.image_ssh_user,
+                self.keypair['private_key'],
+            )
+            for host_route in self.per_test_host_routes.copy():
+                self.del_host_route(self.per_test_host_routes, host_route)

playbooks/plugin-ovn-scenario-pre-run.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - docker-setup

zuul.d/master_jobs.yaml

@@ -637,9 +637,11 @@
     name: neutron-tempest-plugin-ovn
     parent: neutron-tempest-plugin-base-nested-switch
     timeout: 10800
+    pre-run: playbooks/plugin-ovn-scenario-pre-run.yaml
     vars:
       network_api_extensions_ovn:
         - vlan-transparent
+        - external-gateway-multihoming
       # TODO(jlibosva): Remove the NetworkWritableMtuTest test from the list
       # once east/west fragmentation is supported in core OVN
       tempest_exclude_regex: "\
@@ -701,6 +703,8 @@
           /$NEUTRON_CORE_PLUGIN_CONF:
             ml2:
               type_drivers: local,flat,vlan,geneve
+            ml2_type_vlan:
+              network_vlan_ranges: foo:1:10,public
         test-config:
           $TEMPEST_CONFIG:
             network-feature-enabled: