From 3247000865f526149cb000cb1c3af261fcea1c60 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Fri, 25 Sep 2020 15:49:57 +0200 Subject: [PATCH] Add faults test which destroys and creates physical bridges Test is destroying physical bridges on the node where router is hosted so that floating IP shouldn't work at all. Then it creates bridges again and checks if neutron-ovs-agent was able to properly recover from that and FIP will work again. Unfortunatelly this test can't work in e.g. devstack based u/s jobs as there is additional bridge br-infra which provides connection between nodes and this br-infra bridge is connected with br-ex by patch port. When tobiko deletes br-ex and creates it again, there is this patch port to br-infra missing and test is failing alwasy. Test should works fine on deployments based on TripleO. Change-Id: I125285f5f8abd2b2a285a7e46c954d6180d3ec32 --- tobiko/openstack/neutron/__init__.py | 1 + tobiko/openstack/neutron/_agent.py | 4 + .../faults/agents/test_neutron_agents.py | 8 +- tobiko/tests/faults/neutron/__init__.py | 0 tobiko/tests/faults/neutron/test_ovs.py | 91 +++++++++++++++++++ 5 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 tobiko/tests/faults/neutron/__init__.py create mode 100644 tobiko/tests/faults/neutron/test_ovs.py diff --git a/tobiko/openstack/neutron/__init__.py b/tobiko/openstack/neutron/__init__.py index ca17c2651..4dc39a285 100644 --- a/tobiko/openstack/neutron/__init__.py +++ b/tobiko/openstack/neutron/__init__.py @@ -45,6 +45,7 @@ NoSuchNetwork = _client.NoSuchNetwork NoSuchPort = _client.NoSuchPort NoSuchRouter = _client.NoSuchRouter NoSuchSubnet = _client.NoSuchSubnet +AgentNotFoundOnHost = _agent.AgentNotFoundOnHost new_ipv4_cidr = _cidr.new_ipv4_cidr new_ipv6_cidr = _cidr.new_ipv6_cidr diff --git a/tobiko/openstack/neutron/_agent.py b/tobiko/openstack/neutron/_agent.py index ad9d73910..0e94202b2 100644 --- a/tobiko/openstack/neutron/_agent.py +++ b/tobiko/openstack/neutron/_agent.py @@ -17,6 +17,10 @@ import tobiko from tobiko.openstack.neutron import _client +class AgentNotFoundOnHost(tobiko.TobikoException): + message = ("Agent {agent_type!s} not found on the host {host!s}") + + class NetworkingAgentFixture(tobiko.SharedFixture): client = None diff --git a/tobiko/tests/faults/agents/test_neutron_agents.py b/tobiko/tests/faults/agents/test_neutron_agents.py index 520338a9e..cf00c3e18 100644 --- a/tobiko/tests/faults/agents/test_neutron_agents.py +++ b/tobiko/tests/faults/agents/test_neutron_agents.py @@ -31,10 +31,6 @@ from tobiko.shell import sh LOG = log.getLogger(__name__) -class AgentNotFoundOnHost(tobiko.TobikoException): - message = ("Agent {agent_type!s} not found on the host {host!s}") - - class AgentTestMixin(object): def stop_service_on_agents(self, service_name, agents): @@ -350,8 +346,8 @@ class OvsAgentTest(testtools.TestCase, AgentTestMixin): for agent in self.ovs_agents: if host_shortname == tobiko.get_short_hostname(agent['host']): return agent - raise AgentNotFoundOnHost(agent_type="neutron-ovs-agent", - host=host.name) + raise neutron.AgentNotFoundOnHost(agent_type="neutron-ovs-agent", + host=host.name) def test_vm_reachability_during_stop_ovs_agent(self): # Check if vm is reachable before test diff --git a/tobiko/tests/faults/neutron/__init__.py b/tobiko/tests/faults/neutron/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tobiko/tests/faults/neutron/test_ovs.py b/tobiko/tests/faults/neutron/test_ovs.py new file mode 100644 index 000000000..c702f4796 --- /dev/null +++ b/tobiko/tests/faults/neutron/test_ovs.py @@ -0,0 +1,91 @@ +# Copyright (c) 2020 Red Hat +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import collections + +import testtools + +import tobiko +from tobiko.openstack import neutron +from tobiko.openstack import stacks +from tobiko.openstack import topology +from tobiko.shell import ping +from tobiko.shell import sh +from tobiko.tripleo import undercloud + + +@neutron.skip_if_missing_networking_agents(binary='neutron-openvswitch-agent') +@neutron.skip_if_missing_networking_agents(binary='neutron-l3-agent') +class OpenvswitchTest(testtools.TestCase): + + stack = tobiko.required_setup_fixture(stacks.CirrosServerStackFixture) + + def setUp(self): + super(OpenvswitchTest, self).setUp() + self.ovs_agents = neutron.list_agents(agent_type="Open vSwitch agent") + self.router_id = self.stack.network_stack.gateway_id + + self.deleted_bridges = collections.defaultdict(set) + + def tearDown(self): + super(OpenvswitchTest, self).tearDown() + # Try to create all bridges which were deleted during the tests + self._create_bridges() + + def _create_bridges(self): + for host, bridges in self.deleted_bridges.items(): + self._create_bridge(host, bridges) + + def _create_bridge(self, hostname, bridges): + for br_name in bridges: + agent_host = topology.get_openstack_node(hostname=hostname) + sh.execute( + "sudo ovs-vsctl --may-exist add-br %s" % br_name, + ssh_client=agent_host.ssh_client) + + def _delete_bridges(self, hostname, bridges): + for br_name in bridges: + agent_host = topology.get_openstack_node(hostname=hostname) + sh.execute( + "sudo ovs-vsctl del-br %s" % br_name, + ssh_client=agent_host.ssh_client) + self.deleted_bridges[hostname].add(br_name) + + def _get_agent_from_host(self, hostname): + host_shortname = tobiko.get_short_hostname(hostname) + for agent in self.ovs_agents: + if host_shortname == tobiko.get_short_hostname(agent['host']): + return agent + raise neutron.AgentNotFoundOnHost(agent_type="neutron-ovs-agent", + host=hostname) + + @undercloud.skip_if_missing_undercloud + def test_recreate_physical_bridge(self): + # Check if vm is reachable before test + ping.ping_until_received(self.stack.ip_address).assert_replied() + + network_l3_agents = neutron.list_l3_agent_hosting_routers( + self.router_id) + for agent in network_l3_agents: + # Get neutron-ovs-agent bridge mappings + ovs_agent = self._get_agent_from_host(agent['host']) + self._delete_bridges( + agent['host'], + ovs_agent['configurations']['bridge_mappings'].values()) + + ping.ping_until_unreceived(self.stack.ip_address).assert_not_replied() + self._create_bridges() + ping.ping_until_received(self.stack.ip_address).assert_replied()