From 94533e5d7691271abfccb645250678fc7f9c3279 Mon Sep 17 00:00:00 2001
From: German Eichberger <German.eichberger@rackspace.com>
Date: Tue, 31 Jul 2018 17:41:12 +0200
Subject: [PATCH] Delete zombie amphorae when detected

Zombie amphorae will be deleted by the system to lighten the load on
operators if the amphora record is still in the database.

Story: 2003912
Task: 26800

Change-Id: If133a3d36a9381bcca9f7d00f5c1531885907940
(cherry picked from commit df17903ab8cc8edc8da440dea6583ad1c56ffdbe)
---
 .../healthmanager/health_drivers/update_db.py | 26 ++++++++++++++++---
 .../health_drivers/test_update_db.py          | 11 ++++++++
 .../notes/zombie_amp-1b435eb66643dab8.yaml    | 12 +++++++++
 3 files changed, 45 insertions(+), 4 deletions(-)
 create mode 100644 releasenotes/notes/zombie_amp-1b435eb66643dab8.yaml

diff --git a/octavia/controller/healthmanager/health_drivers/update_db.py b/octavia/controller/healthmanager/health_drivers/update_db.py
index 208f1ce996..0827440524 100644
--- a/octavia/controller/healthmanager/health_drivers/update_db.py
+++ b/octavia/controller/healthmanager/health_drivers/update_db.py
@@ -176,10 +176,28 @@ class UpdateHealthDb(update_base.HealthUpdateBase):
                 LOG.debug('Received a health heartbeat from amphora {0} with '
                           'IP {1} that should not exist. This amphora may be '
                           'in the process of being deleted, in which case you '
-                          'will only see this message a few times. However if '
-                          'it is repeating this amphora should be manually '
-                          'deleted.'.format(health['id'], srcaddr))
-                return
+                          'will only see this message a few '
+                          'times'.format(health['id'], srcaddr))
+                if not amp:
+                    LOG.warning('The amphora {0} with IP {1} is missing from '
+                                'the DB, so it cannot be automatically '
+                                'deleted (the compute_id is unknown). An '
+                                'operator must manually delete it from the '
+                                'compute service.'.format(health['id'],
+                                                          srcaddr))
+                    return
+                # delete the amp right there
+                try:
+                    compute = stevedore_driver.DriverManager(
+                        namespace='octavia.compute.drivers',
+                        name=CONF.controller_worker.compute_driver,
+                        invoke_on_load=True
+                    ).driver
+                    compute.delete(amp.compute_id)
+                    return
+                except Exception as e:
+                    LOG.info("Error deleting amp {0} with IP {1}".format(
+                        health['id'], srcaddr), e)
             expected_listener_count = 0
 
         listeners = health['listeners']
diff --git a/octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py b/octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py
index 54ff5972fb..2e60c8771c 100644
--- a/octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py
+++ b/octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py
@@ -1021,6 +1021,17 @@ class TestUpdateHealthDb(base.TestCase):
                         self.session_mock, member_id,
                         operating_status=constants.ONLINE)
 
+    @mock.patch('stevedore.driver.DriverManager.driver')
+    def test_update_health_zombie(self, mock_driver):
+        health = {"id": self.FAKE_UUID_1, "listeners": {}}
+
+        self.amphora_repo.get_lb_for_health_update.return_value = None
+        amp_mock = mock.MagicMock()
+        self.amphora_repo.get.return_value = amp_mock
+        self.hm.update_health(health, '192.0.2.1')
+        mock_driver.delete.assert_called_once_with(
+            amp_mock.compute_id)
+
     def test_update_health_no_status_change(self):
         health = {
             "id": self.FAKE_UUID_1,
diff --git a/releasenotes/notes/zombie_amp-1b435eb66643dab8.yaml b/releasenotes/notes/zombie_amp-1b435eb66643dab8.yaml
new file mode 100644
index 0000000000..9c174e4eac
--- /dev/null
+++ b/releasenotes/notes/zombie_amp-1b435eb66643dab8.yaml
@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    This will automatically nova delete zombie amphora when they
+    are detected by Octavia. Zombie amphorae are amphorae which
+    report health messages but appear DELETED in Octavia's
+    database.
+other:
+  - |
+    Processing zombie amphora is already expensive and this adds
+    another step which could increase the load on Octavia Health
+    Manager, especially during Nova API slowness.