Catch error deleting trust on stack delete

When deleting a stack, it's possible for deleting the trust to fail,
for example if the user deleting the stack is not the user who created
it, or an admin (which raises a Forbidden error), or due to some
other transient error e.g connection to keystone interrupted.

Currently in this case, we fail to mark the stack deleted in the DB
and leave the status "DELETE, COMPLETE", which is misleading.

Change-Id: Ie8e9ea48bc4f44e56ff4764123fcca733f5bd458
Partial-Bug: #1247200

heat/engine/parser.py

@@ -569,14 +569,22 @@ class Stack(collections.Mapping):
             stack_status = self.FAILED
             reason = '%s timed out' % action.title()
 
-        self.state_set(action, stack_status, reason)
         if stack_status != self.FAILED:
             # If we created a trust, delete it
             stack = db_api.stack_get(self.context, self.id)
             user_creds = db_api.user_creds_get(stack.user_creds_id)
             trust_id = user_creds.get('trust_id')
             if trust_id:
-                self.clients.keystone().delete_trust(trust_id)
+                try:
+                    self.clients.keystone().delete_trust(trust_id)
+                except Exception as ex:
+                    logger.exception(ex)
+                    stack_status = self.FAILED
+                    reason = "Error deleting trust: %s" % str(ex)
+
+        self.state_set(action, stack_status, reason)
+
+        if stack_status != self.FAILED:
             # delete the stack
             db_api.stack_delete(self.context, self.id)
             self.id = None

heat/tests/test_parser.py

@@ -17,6 +17,8 @@ import time
 
 import testscenarios
 
+from keystoneclient import exceptions as kc_exceptions
+
 from oslo.config import cfg
 
 from heat.engine import environment
@@ -833,6 +835,34 @@ class StackTest(HeatTestCase):
         self.assertEqual(self.stack.state,
                          (parser.Stack.DELETE, parser.Stack.COMPLETE))
 
+    @utils.stack_delete_after
+    def test_delete_trust_fail(self):
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        class FakeKeystoneClientFail(FakeKeystoneClient):
+            def delete_trust(self, trust_id):
+                raise kc_exceptions.Forbidden("Denied!")
+
+        self.m.StubOutWithMock(clients.OpenStackClients, 'keystone')
+        clients.OpenStackClients.keystone().MultipleTimes().AndReturn(
+            FakeKeystoneClientFail())
+        self.m.ReplayAll()
+
+        self.stack = parser.Stack(
+            self.ctx, 'delete_trust', template.Template({}))
+        stack_id = self.stack.store()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertIsNotNone(db_s)
+
+        self.stack.delete()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertIsNotNone(db_s)
+        self.assertEqual(self.stack.state,
+                         (parser.Stack.DELETE, parser.Stack.FAILED))
+        self.assertIn('Error deleting trust', self.stack.status_reason)
+
     @utils.stack_delete_after
     def test_suspend_resume(self):
         self.m.ReplayAll()