Merge "Allow update cluster status with admin context"

magnum/drivers/common/driver.py

@@ -141,7 +141,7 @@ class Driver(object):
             context, cluster.cluster_template_id)
         return cls.get_driver(ct.server_type, ct.cluster_distro, ct.coe)
 
-    def update_cluster_status(self, context, cluster):
+    def update_cluster_status(self, context, cluster, use_admin_ctx=False):
         """Update the cluster status based on underlying orchestration
 
            This is an optional method if your implementation does not need

magnum/drivers/heat/driver.py

@@ -120,13 +120,16 @@ class HeatDriver(driver.Driver):
             osc = clients.OpenStackClients(context)
             self._delete_stack(context, osc, nodegroup.stack_id)
 
-    def update_cluster_status(self, context, cluster):
+    def update_cluster_status(self, context, cluster, use_admin_ctx=False):
         if cluster.stack_id is None:
             # NOTE(mgoddard): During cluster creation it is possible to poll
             # the cluster before its heat stack has been created. See bug
             # 1682058.
             return
-        stack_ctx = mag_ctx.make_cluster_context(cluster)
+        if use_admin_ctx:
+            stack_ctx = context
+        else:
+            stack_ctx = mag_ctx.make_cluster_context(cluster)
         poller = HeatPoller(clients.OpenStackClients(stack_ctx), context,
                             cluster, self)
         poller.poll_and_check()

magnum/service/periodic.py

@@ -23,6 +23,7 @@ from oslo_service import periodic_task
 from pycadf import cadftaxonomy as taxonomy
 
 from magnum.common import context
+from magnum.common import exception
 from magnum.common import profiler
 from magnum.common import rpc
 from magnum.conductor import monitors
@@ -68,7 +69,19 @@ class ClusterUpdateJob(object):
         # get the driver for the cluster
         cdriver = driver.Driver.get_driver_for_cluster(self.ctx, self.cluster)
         # ask the driver to sync status
-        cdriver.update_cluster_status(self.ctx, self.cluster)
+        try:
+            cdriver.update_cluster_status(self.ctx, self.cluster)
+        except exception.AuthorizationFailure as e:
+            trust_ex = ("Could not find trust: %s" % self.cluster.trust_id)
+            # Try to use admin context if trust not found.
+            # This will make sure even with trust got deleted out side of
+            # Magnum, we still be able to check cluster status
+            if trust_ex in str(e):
+                cdriver.update_cluster_status(
+                    self.ctx, self.cluster, use_admin_ctx=True)
+            else:
+                raise
+
         LOG.debug("Status for cluster %s updated to %s (%s)",
                   self.cluster.id, self.cluster.status,
                   self.cluster.status_reason)

magnum/tests/unit/service/test_periodic.py

@@ -17,6 +17,7 @@ from unittest import mock
 from oslo_utils import uuidutils
 
 from magnum.common import context
+from magnum.common import exception
 from magnum.common.rpc_service import CONF
 from magnum.db.sqlalchemy import api as dbapi
 from magnum.drivers.common import driver
@@ -189,6 +190,32 @@ class PeriodicTestCase(base.TestCase):
         self.mock_driver.update_cluster_status.side_effect = (
             _mock_update_status)
 
+    @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')
+    def test_update_status_non_trusts_error(self, mock_get_driver):
+        mock_get_driver.return_value = self.mock_driver
+        trust_ex = ("Unknown Keystone error")
+        self.mock_driver.update_cluster_status.side_effect = \
+            exception.AuthorizationFailure(client='keystone', message=trust_ex)
+        self.assertRaises(
+            exception.AuthorizationFailure,
+            periodic.ClusterUpdateJob(
+                self.context, self.cluster1).update_status
+        )
+        self.assertEqual(1, self.mock_driver.update_cluster_status.call_count)
+
+    @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')
+    def test_update_status_trusts_not_found(self, mock_get_driver):
+        mock_get_driver.return_value = self.mock_driver
+        trust_ex = ("Could not find trust: %s" % self.cluster1.trust_id)
+        self.mock_driver.update_cluster_status.side_effect = \
+            exception.AuthorizationFailure(client='keystone', message=trust_ex)
+        self.assertRaises(
+            exception.AuthorizationFailure,
+            periodic.ClusterUpdateJob(
+                self.context, self.cluster1).update_status
+        )
+        self.assertEqual(2, self.mock_driver.update_cluster_status.call_count)
+
     @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall',
                 new=fakes.FakeLoopingCall)
     @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')