Merge "Delete zombie amphorae when detected" into stable/rocky
This commit is contained in:
commit
cc85cede24
|
@ -176,10 +176,28 @@ class UpdateHealthDb(update_base.HealthUpdateBase):
|
|||
LOG.debug('Received a health heartbeat from amphora {0} with '
|
||||
'IP {1} that should not exist. This amphora may be '
|
||||
'in the process of being deleted, in which case you '
|
||||
'will only see this message a few times. However if '
|
||||
'it is repeating this amphora should be manually '
|
||||
'deleted.'.format(health['id'], srcaddr))
|
||||
return
|
||||
'will only see this message a few '
|
||||
'times'.format(health['id'], srcaddr))
|
||||
if not amp:
|
||||
LOG.warning('The amphora {0} with IP {1} is missing from '
|
||||
'the DB, so it cannot be automatically '
|
||||
'deleted (the compute_id is unknown). An '
|
||||
'operator must manually delete it from the '
|
||||
'compute service.'.format(health['id'],
|
||||
srcaddr))
|
||||
return
|
||||
# delete the amp right there
|
||||
try:
|
||||
compute = stevedore_driver.DriverManager(
|
||||
namespace='octavia.compute.drivers',
|
||||
name=CONF.controller_worker.compute_driver,
|
||||
invoke_on_load=True
|
||||
).driver
|
||||
compute.delete(amp.compute_id)
|
||||
return
|
||||
except Exception as e:
|
||||
LOG.info("Error deleting amp {0} with IP {1}".format(
|
||||
health['id'], srcaddr), e)
|
||||
expected_listener_count = 0
|
||||
|
||||
listeners = health['listeners']
|
||||
|
|
|
@ -1021,6 +1021,17 @@ class TestUpdateHealthDb(base.TestCase):
|
|||
self.session_mock, member_id,
|
||||
operating_status=constants.ONLINE)
|
||||
|
||||
@mock.patch('stevedore.driver.DriverManager.driver')
|
||||
def test_update_health_zombie(self, mock_driver):
|
||||
health = {"id": self.FAKE_UUID_1, "listeners": {}}
|
||||
|
||||
self.amphora_repo.get_lb_for_health_update.return_value = None
|
||||
amp_mock = mock.MagicMock()
|
||||
self.amphora_repo.get.return_value = amp_mock
|
||||
self.hm.update_health(health, '192.0.2.1')
|
||||
mock_driver.delete.assert_called_once_with(
|
||||
amp_mock.compute_id)
|
||||
|
||||
def test_update_health_no_status_change(self):
|
||||
health = {
|
||||
"id": self.FAKE_UUID_1,
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
---
|
||||
fixes:
|
||||
- |
|
||||
This will automatically nova delete zombie amphora when they
|
||||
are detected by Octavia. Zombie amphorae are amphorae which
|
||||
report health messages but appear DELETED in Octavia's
|
||||
database.
|
||||
other:
|
||||
- |
|
||||
Processing zombie amphora is already expensive and this adds
|
||||
another step which could increase the load on Octavia Health
|
||||
Manager, especially during Nova API slowness.
|
Loading…
Reference in New Issue