Merge "Implement locking in abandon stack"

heat/engine/service.py

@@ -735,13 +735,29 @@ class EngineService(service.Service):
         st = self._get_stack(cnxt, stack_identity)
         logger.info(_('abandoning stack %s') % st.name)
         stack = parser.Stack.load(cnxt, stack=st)
+        lock = stack_lock.StackLock(cnxt, stack, self.engine_id)
+        acquire_result = lock.try_acquire()
 
-        # Get stack details before deleting it.
-        stack_info = stack.get_abandon_data()
-        # Set deletion policy to 'Retain' for all resources in the stack.
-        stack.set_deletion_policy(resource.RETAIN)
-        self.thread_group_mgr.start_with_lock(cnxt, stack, self.engine_id,
-                                              stack.delete)
+        # If an action is in progress, 'try_acquire' returns engine UUID. If
+        # the returned engine is alive, then throw ActionInProgress exception
+        if (acquire_result and
+                (acquire_result == self.engine_id or
+                 stack_lock.StackLock.engine_alive(cnxt, acquire_result))):
+            raise exception.ActionInProgress(stack_name=stack.name,
+                                             action=stack.action)
+
+        try:
+            # Get stack details before deleting it.
+            stack_info = stack.get_abandon_data()
+            # Set deletion policy to 'Retain' for all resources in the stack.
+            stack.set_deletion_policy(resource.RETAIN)
+        except:
+            with excutils.save_and_reraise_exception():
+                lock.release(stack.id)
+
+        self.thread_group_mgr.start_with_acquired_lock(stack,
+                                                       lock,
+                                                       stack.delete)
         return stack_info
 
     def list_resource_types(self, cnxt, support_status=None):

heat/tests/test_engine_service.py

@@ -1748,6 +1748,76 @@ class StackServiceTest(HeatTestCase):
         self.m.StubOutWithMock(parser.Stack, 'load')
         parser.Stack.load(self.ctx,
                           stack=mox.IgnoreArg()).AndReturn(self.stack)
+        expected_res = {
+            u'WebServer': {
+                'action': 'CREATE',
+                'metadata': {},
+                'name': u'WebServer',
+                'resource_data': {},
+                'resource_id': 9999,
+                'status': 'COMPLETE',
+                'type': u'AWS::EC2::Instance'}}
+        self.m.StubOutWithMock(stack_lock.StackLock, 'try_acquire')
+        stack_lock.StackLock.try_acquire().AndReturn(None)
+        self.m.ReplayAll()
+        ret = self.eng.abandon_stack(self.ctx, self.stack.identifier())
+        self.assertEqual(6, len(ret))
+        self.assertEqual('CREATE', ret['action'])
+        self.assertEqual('COMPLETE', ret['status'])
+        self.assertEqual('service_abandon_stack', ret['name'])
+        self.assertIn('id', ret)
+        self.assertEqual(expected_res, ret['resources'])
+        self.assertEqual(self.stack.t.t, ret['template'])
+        self.m.VerifyAll()
+
+    @stack_context('service_abandon_stack')
+    def test_abandon_stack_fails_action_in_progress_on_other_engine(self):
+        self.m.StubOutWithMock(parser.Stack, 'load')
+        parser.Stack.load(self.ctx,
+                          stack=mox.IgnoreArg()).AndReturn(self.stack)
+        self.m.StubOutWithMock(stack_lock.StackLock, 'try_acquire')
+        stack_lock.StackLock.try_acquire().AndReturn("other-engine-fake-uuid")
+        self.m.StubOutWithMock(stack_lock.StackLock, 'engine_alive')
+        stack_lock.StackLock.engine_alive(
+            self.ctx,
+            "other-engine-fake-uuid").AndReturn(True)
+
+        self.m.ReplayAll()
+        ex = self.assertRaises(rpc_common.ClientException,
+                               self.eng.abandon_stack,
+                               self.ctx,
+                               self.stack.identifier())
+        self.assertEqual(ex._exc_info[0], exception.ActionInProgress)
+        self.m.VerifyAll()
+
+    @stack_context('service_abandon_stack')
+    def test_abandon_stack_fails_action_in_progress_on_same_engine(self):
+        self.m.StubOutWithMock(parser.Stack, 'load')
+        parser.Stack.load(self.ctx,
+                          stack=mox.IgnoreArg()).AndReturn(self.stack)
+        self.m.StubOutWithMock(stack_lock.StackLock, 'try_acquire')
+        stack_lock.StackLock.try_acquire().AndReturn(self.eng.engine_id)
+
+        self.m.ReplayAll()
+        ex = self.assertRaises(rpc_common.ClientException,
+                               self.eng.abandon_stack,
+                               self.ctx,
+                               self.stack.identifier())
+        self.assertEqual(ex._exc_info[0], exception.ActionInProgress)
+        self.m.VerifyAll()
+
+    @stack_context('service_abandon_stack')
+    def test_abandon_stack_success_other_engine_not_alive(self):
+        self.m.StubOutWithMock(parser.Stack, 'load')
+        parser.Stack.load(self.ctx,
+                          stack=mox.IgnoreArg()).AndReturn(self.stack)
+        self.m.StubOutWithMock(stack_lock.StackLock, 'try_acquire')
+        stack_lock.StackLock.try_acquire().AndReturn("other-engine-fake-uuid")
+        self.m.StubOutWithMock(stack_lock.StackLock, 'engine_alive')
+        stack_lock.StackLock.engine_alive(
+            self.ctx,
+            "other-engine-fake-uuid").AndReturn(False)
+
         expected_res = {
             u'WebServer': {
                 'action': 'CREATE',