Ensure kuryr-controller recover from lb in ERROR status

This patch ensures kuryr controller can recover from the situation
when the created lb goes into ERROR status (instead of ACTIVE). Now,
when the kuryr-controller finds the created lb, it checks its
provisioning_status and if in ERROR status it will delete it and
ensure a ResourceNotReady exception is triggered so that a new
creation action is triggered after the deletion of the lb in ERROR
status.

Closes-Bug: 1815880
Change-Id: I3f5de710a5ff37dee05f5f8826cb37c343141a08
(cherry picked from commit 33594b8702)

kuryr_kubernetes/controller/drivers/lbaasv2.py

@@ -56,6 +56,8 @@ class LBaaSv2Driver(base.LBaaSDriver):
         if not response:
             # NOTE(ivc): load balancer was present before 'create', but got
             # deleted externally between 'create' and 'find'
+            # NOTE(ltomasbo): or it is in ERROR status, so we deleted and
+            # trigger the retry
             raise k_exc.ResourceNotReady(request)
 
         try:
@@ -259,6 +261,10 @@ class LBaaSv2Driver(base.LBaaSDriver):
             loadbalancer.id = response['loadbalancers'][0]['id']
             loadbalancer.port_id = self._get_vip_port_id(loadbalancer)
             loadbalancer.provider = response['loadbalancers'][0]['provider']
+            if (response['loadbalancers'][0]['provisioning_status'] ==
+                    'ERROR'):
+                self.release_loadbalancer(loadbalancer)
+                return None
         except (KeyError, IndexError):
             return None
 

kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py

@@ -275,7 +275,8 @@ class TestLBaaSv2Driver(test_base.TestCase):
             provider='haproxy', security_groups=[])
         loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C'
         resp = {'loadbalancers': [{'id': loadbalancer_id,
-                                   'provider': 'haproxy'}]}
+                                   'provider': 'haproxy',
+                                   'provisioning_status': 'ACTIVE'}]}
         lbaas.list_loadbalancers.return_value = resp
 
         ret = cls._find_loadbalancer(m_driver, loadbalancer)
@@ -288,6 +289,7 @@ class TestLBaaSv2Driver(test_base.TestCase):
             self.assertEqual(getattr(loadbalancer, attr),
                              getattr(ret, attr))
         self.assertEqual(loadbalancer_id, ret.id)
+        m_driver.release_loadbalancer.assert_not_called()
 
     def test_find_loadbalancer_not_found(self):
         lbaas = self.useFixture(k_fix.MockLBaaSClient()).client
@@ -306,6 +308,29 @@ class TestLBaaSv2Driver(test_base.TestCase):
             vip_address=str(loadbalancer.ip),
             vip_subnet_id=loadbalancer.subnet_id)
         self.assertIsNone(ret)
+        m_driver.release_loadbalancer.assert_not_called()
+
+    def test_find_loadbalancer_error(self):
+        lbaas = self.useFixture(k_fix.MockLBaaSClient()).client
+        cls = d_lbaasv2.LBaaSv2Driver
+        m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver)
+        loadbalancer = obj_lbaas.LBaaSLoadBalancer(
+            name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4',
+            subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1')
+        loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C'
+        resp = {'loadbalancers': [{'id': loadbalancer_id,
+                                   'provider': 'haproxy',
+                                   'provisioning_status': 'ERROR'}]}
+        lbaas.list_loadbalancers.return_value = resp
+
+        ret = cls._find_loadbalancer(m_driver, loadbalancer)
+        lbaas.list_loadbalancers.assert_called_once_with(
+            name=loadbalancer.name,
+            project_id=loadbalancer.project_id,
+            vip_address=str(loadbalancer.ip),
+            vip_subnet_id=loadbalancer.subnet_id)
+        self.assertIsNone(ret)
+        m_driver.release_loadbalancer.assert_called_once()
 
     def test_create_listener(self):
         lbaas = self.useFixture(k_fix.MockLBaaSClient()).client