From 503e3949f895c393c6ed2e2bdeb881f9fa299ad3 Mon Sep 17 00:00:00 2001
From: Carlos Goncalves <cgoncalves@redhat.com>
Date: Fri, 22 Feb 2019 23:34:44 +0000
Subject: [PATCH] Fix LB failover when in ERROR

Load balancers in ERROR provisioning status could not be failed over.

One possible scenario where LBs go into ERROR is when services are
started before compute nodes are up on a cluster reboot:

- Octavia services are started
- Health Manager does not receive heartbeat and triggers failover
- Failover fails due to lack of availabble compute nodes to spawn
  amphora VMs

Story: 2005078
Task: 29657

Change-Id: Ic4b4516cd6b2a254ea32939668c906486066da42
---
 octavia/api/v2/controllers/load_balancer.py          | 12 +++++++++++-
 .../tests/functional/api/v2/test_load_balancer.py    |  2 +-
 .../fix-lb-error-failover-2c17afaa20c0c97f.yaml      |  5 +++++
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/fix-lb-error-failover-2c17afaa20c0c97f.yaml

diff --git a/octavia/api/v2/controllers/load_balancer.py b/octavia/api/v2/controllers/load_balancer.py
index 34afebb166..4fc49c495c 100644
--- a/octavia/api/v2/controllers/load_balancer.py
+++ b/octavia/api/v2/controllers/load_balancer.py
@@ -106,6 +106,16 @@ class LoadBalancersController(base.BaseController):
             raise exceptions.LBPendingStateError(
                 state=prov_status, id=id)
 
+    def _test_and_set_failover_prov_status(self, session, id):
+        lb_repo = self.repositories.load_balancer
+        if not lb_repo.set_status_for_failover(session, id,
+                                               constants.PENDING_UPDATE):
+            prov_status = lb_repo.get(session, id=id).provisioning_status
+            LOG.info("Invalid state %(state)s of loadbalancer resource %(id)s",
+                     {"state": prov_status, "id": id})
+            raise exceptions.LBPendingStateError(
+                state=prov_status, id=id)
+
     @staticmethod
     def _validate_network_and_fill_or_validate_subnet(load_balancer):
         network = validate.network_exists_optionally_contains_subnet(
@@ -717,7 +727,7 @@ class FailoverController(LoadBalancersController):
         driver = driver_factory.get_driver(db_lb.provider)
 
         with db_api.get_lock_session() as lock_session:
-            self._test_lb_status(lock_session, self.lb_id)
+            self._test_and_set_failover_prov_status(lock_session, self.lb_id)
             LOG.info("Sending failover request for load balancer %s to the "
                      "provider %s", self.lb_id, driver.name)
             driver_utils.call_provider(
diff --git a/octavia/tests/functional/api/v2/test_load_balancer.py b/octavia/tests/functional/api/v2/test_load_balancer.py
index 285d2d3245..17493b74df 100644
--- a/octavia/tests/functional/api/v2/test_load_balancer.py
+++ b/octavia/tests/functional/api/v2/test_load_balancer.py
@@ -1955,7 +1955,7 @@ class TestLoadBalancer(base.BaseAPITest):
                                 status=constants.ERROR)
         self.app.put(self._get_full_path(
             self.LB_PATH.format(lb_id=lb_dict.get('id')) + "/failover"),
-            status=409)
+            status=202)
 
     def test_failover_not_authorized(self):
         project_id = uuidutils.generate_uuid()
diff --git a/releasenotes/notes/fix-lb-error-failover-2c17afaa20c0c97f.yaml b/releasenotes/notes/fix-lb-error-failover-2c17afaa20c0c97f.yaml
new file mode 100644
index 0000000000..0fe4c856e3
--- /dev/null
+++ b/releasenotes/notes/fix-lb-error-failover-2c17afaa20c0c97f.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fix load balancers that could not be failed over when in ERROR provisioning
+    status.