Merge "Fix amphora failover when VRRP port is missing" into stable/train

This commit is contained in:
Zuul 2021-01-21 11:33:31 +00:00 committed by Gerrit Code Review
commit fd622009e6
5 changed files with 98 additions and 25 deletions

View File

@ -128,19 +128,20 @@ class ListenersStart(BaseAmphoraTask):
class AmphoraIndexListenersReload(BaseAmphoraTask):
"""Task to reload all listeners on an amphora."""
def execute(self, loadbalancer, amphorae, amphora_index,
def execute(self, loadbalancer, amphora_index, amphorae,
timeout_dict=None):
"""Execute listener reload routines for listeners on an amphora."""
if loadbalancer.listeners:
self.amphora_driver.reload(
loadbalancer, amphorae[amphora_index], timeout_dict)
def revert(self, loadbalancer, *args, **kwargs):
"""Handle failed listeners reloads."""
LOG.warning("Reverting listener reload.")
for listener in loadbalancer.listeners:
self.task_utils.mark_listener_prov_status_error(listener.id)
try:
self.amphora_driver.reload(
loadbalancer, amphorae[amphora_index], timeout_dict)
except Exception as e:
amphora_id = amphorae[amphora_index].id
LOG.warning('Failed to reload listeners on amphora %s. '
'Skipping this amphora as it is failing to '
'reload due to: %s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
class ListenerDelete(BaseAmphoraTask):
@ -310,7 +311,7 @@ class AmphoraUpdateVRRPInterface(BaseAmphoraTask):
class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask):
"""Task to get and update the VRRP interface device name from amphora."""
def execute(self, amphorae, amphora_index, timeout_dict=None):
def execute(self, amphora_index, amphorae, timeout_dict=None):
amphora_id = amphorae[amphora_index].id
try:
interface = self.amphora_driver.get_interface_from_ip(
@ -378,7 +379,7 @@ class AmphoraIndexVRRPUpdate(BaseAmphoraTask):
'to: %s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
return
LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id)
@ -409,8 +410,17 @@ class AmphoraIndexVRRPStart(BaseAmphoraTask):
"""
def execute(self, amphora_index, amphorae, timeout_dict=None):
self.amphora_driver.start_vrrp_service(amphorae[amphora_index],
timeout_dict)
amphora_id = amphorae[amphora_index].id
try:
self.amphora_driver.start_vrrp_service(amphorae[amphora_index],
timeout_dict)
except Exception as e:
LOG.error('Failed to start VRRP on amphora %s. '
'Skipping this amphora as it is failing to start due '
'to: %s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
return
LOG.debug("Started VRRP on amphora %s.", amphorae[amphora_index].id)

View File

@ -715,8 +715,13 @@ class AllowedAddressPairsDriver(neutron_base.BaseNeutronDriver):
vip_subnet, vip_port)
else:
for amp in loadbalancer.amphorae:
self._get_amp_net_configs(amp, amp_configs,
vip_subnet, vip_port)
try:
self._get_amp_net_configs(amp, amp_configs,
vip_subnet, vip_port)
except Exception as e:
LOG.warning('Getting network configurations for amphora '
'%(amp)s failed due to %(err)s.',
{'amp': amp.id, 'err': str(e)})
return amp_configs
# TODO(johnsom) This may be dead code now. Remove in failover for v2 patch

View File

@ -210,6 +210,7 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_lb = mock.MagicMock()
mock_listener = mock.MagicMock()
mock_listener.id = '12345'
mock_driver.reload.side_effect = [mock.DEFAULT, Exception('boom')]
# Test no listeners
mock_lb.listeners = None
@ -219,14 +220,19 @@ class TestAmphoraDriverTasks(base.TestCase):
# Test with listeners
mock_driver.start.reset_mock()
mock_lb.listeners = [mock_listener]
listeners_reload_obj.execute(mock_lb, [amphora_mock], 0,
listeners_reload_obj.execute(mock_lb, 0, [amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock,
self.timeout_dict)
# Test revert
mock_lb.listeners = [mock_listener]
listeners_reload_obj.revert(mock_lb)
mock_prov_status_error.assert_called_once_with('12345')
# Test with reload exception
mock_driver.reload.reset_mock()
listeners_reload_obj.execute(mock_lb, 0, [amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock,
self.timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, amphora_mock.id, status=constants.ERROR)
@mock.patch('octavia.controller.worker.task_utils.TaskUtils.'
'mark_listener_prov_status_error')
@ -630,7 +636,7 @@ class TestAmphoraDriverTasks(base.TestCase):
amphora_update_vrrp_interface_obj = (
amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface())
amphora_update_vrrp_interface_obj.execute(
[_amphora_mock], 0, timeout_dict)
0, [_amphora_mock], timeout_dict)
mock_driver.get_interface_from_ip.assert_called_once_with(
_amphora_mock, _amphora_mock.vrrp_ip, timeout_dict=timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
@ -639,7 +645,7 @@ class TestAmphoraDriverTasks(base.TestCase):
# Test with an exception
mock_amphora_repo_update.reset_mock()
amphora_update_vrrp_interface_obj.execute(
[_amphora_mock], 0, timeout_dict)
0, [_amphora_mock], timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _amphora_mock.id, status=constants.ERROR)
@ -740,11 +746,23 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_amphora_repo_update):
amphora_vrrp_start_obj = (
amphora_driver_tasks.AmphoraIndexVRRPStart())
mock_driver.start_vrrp_service.side_effect = [mock.DEFAULT,
Exception('boom')]
amphora_vrrp_start_obj.execute(0, [_amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.start_vrrp_service.assert_called_once_with(
_amphora_mock, self.timeout_dict)
# Test with a start exception
mock_driver.start_vrrp_service.reset_mock()
amphora_vrrp_start_obj.execute(0, [_amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.start_vrrp_service.assert_called_once_with(
_amphora_mock, self.timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _amphora_mock.id, status=constants.ERROR)
def test_amphora_compute_connectivity_wait(self,
mock_driver,
mock_generate_uuid,

View File

@ -1244,12 +1244,19 @@ class TestAllowedAddressPairsDriver(base.TestCase):
def test_get_network_configs(self):
amphora_mock = mock.MagicMock()
amphora2_mock = mock.MagicMock()
load_balancer_mock = mock.MagicMock()
vip_mock = mock.MagicMock()
amphora_mock.status = constants.DELETED
load_balancer_mock.amphorae = [amphora_mock]
show_port = self.driver.neutron_client.show_port
show_port.return_value = t_constants.MOCK_NEUTRON_PORT
show_port.side_effect = [
t_constants.MOCK_NEUTRON_PORT, t_constants.MOCK_NEUTRON_PORT,
t_constants.MOCK_NEUTRON_PORT, t_constants.MOCK_NEUTRON_PORT,
t_constants.MOCK_NEUTRON_PORT, t_constants.MOCK_NEUTRON_PORT,
t_constants.MOCK_NEUTRON_PORT, t_constants.MOCK_NEUTRON_PORT,
t_constants.MOCK_NEUTRON_PORT, t_constants.MOCK_NEUTRON_PORT,
Exception('boom')]
fake_subnet = {'subnet': {
'id': t_constants.MOCK_SUBNET_ID,
'gateway_ip': t_constants.MOCK_IP_ADDRESS,
@ -1266,7 +1273,12 @@ class TestAllowedAddressPairsDriver(base.TestCase):
amphora_mock.vrrp_ip = "10.0.0.1"
amphora_mock.ha_port_id = 3
amphora_mock.ha_ip = "10.0.0.2"
load_balancer_mock.amphorae = [amphora_mock]
amphora2_mock.id = 333
amphora2_mock.status = constants.ACTIVE
amphora2_mock.vrrp_port_id = 3
amphora2_mock.vrrp_ip = "10.0.0.2"
amphora2_mock.ha_port_id = 4
amphora2_mock.ha_ip = "10.0.0.3"
configs = self.driver.get_network_configs(load_balancer_mock)
self.assertEqual(1, len(configs))
@ -1283,6 +1295,29 @@ class TestAllowedAddressPairsDriver(base.TestCase):
self.assertEqual(expected_subnet_id, config.ha_subnet.id)
self.assertEqual(expected_subnet_id, config.vrrp_subnet.id)
# Test with a specific amphora
configs = self.driver.get_network_configs(load_balancer_mock,
amphora_mock)
self.assertEqual(1, len(configs))
config = configs[222]
# TODO(ptoohill): find a way to return different items for multiple
# calls to the same method, right now each call to show subnet
# will return the same values if a method happens to call it
# multiple times for different subnets. We should be able to verify
# different requests get different expected data.
expected_port_id = t_constants.MOCK_NEUTRON_PORT['port']['id']
self.assertEqual(expected_port_id, config.ha_port.id)
self.assertEqual(expected_port_id, config.vrrp_port.id)
expected_subnet_id = fake_subnet['subnet']['id']
self.assertEqual(expected_subnet_id, config.ha_subnet.id)
self.assertEqual(expected_subnet_id, config.vrrp_subnet.id)
# Test with a load balancer with two amphora, one that has a
# neutron problem.
load_balancer_mock.amphorae = [amphora_mock, amphora2_mock]
configs = self.driver.get_network_configs(load_balancer_mock)
self.assertEqual(1, len(configs))
@mock.patch('time.sleep')
def test_wait_for_port_detach(self, mock_sleep):
amphora = data_models.Amphora(

View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixed an issue with failing over an amphora if the pair amphora in an
active/standby pair had a missing VRRP port in neutron.