Rollback to the original status when server powering action failed

As we don't move server status to error when power action failed, the
status will be stuck in *ing forever.

Change-Id: Ie98ec4c5d2d848ca13dae4d891f762cd58650fad
Closes-Bug: #1690096

mogan/common/states.py

@@ -195,6 +195,12 @@ machine.add_transition(SOFT_POWERING_OFF, STOPPED, 'done')
 machine.add_transition(REBOOTING, ACTIVE, 'done')
 machine.add_transition(SOFT_REBOOTING, ACTIVE, 'done')
 
+machine.add_transition(POWERING_ON, STOPPED, 'fail')
+machine.add_transition(POWERING_OFF, ACTIVE, 'fail')
+machine.add_transition(SOFT_POWERING_OFF, ACTIVE, 'fail')
+machine.add_transition(REBOOTING, ACTIVE, 'fail')
+machine.add_transition(SOFT_REBOOTING, ACTIVE, 'fail')
+
 # All unstable states are allowed to transition to ERROR and DELETING
 for state in UNSTABLE_STATES:
     machine.add_transition(state, ERROR, 'error')

mogan/engine/manager.py

@@ -42,6 +42,14 @@ from mogan.scheduler import utils as sched_utils
 
 LOG = log.getLogger(__name__)
 
+POWER_NOTIFICATION_MAP = {
+    'on': fields.NotificationAction.POWER_ON,
+    'off': fields.NotificationAction.POWER_OFF,
+    'reboot': fields.NotificationAction.REBOOT,
+    'soft_off': fields.NotificationAction.SOFT_POWER_OFF,
+    'soft_reboot': fields.NotificationAction.SOFT_REBOOT
+}
+
 
 @utils.expects_func_args('server')
 def wrap_server_fault(function):
@@ -467,6 +475,7 @@ class EngineManager(base_manager.BaseEngineManager):
         server.destroy()
         LOG.info("Deleted server successfully.")
 
+    @wrap_server_fault
     def set_power_state(self, context, server, state):
         """Set power state for the specified server."""
 
@@ -479,9 +488,30 @@ class EngineManager(base_manager.BaseEngineManager):
                        'server': server})
             self.driver.set_power_state(context, server, state)
 
-        do_set_power_state()
-        server.power_state = self.driver.get_power_state(context,
-                                                         server.uuid)
+        try:
+            do_set_power_state()
+            server.power_state = self.driver.get_power_state(context,
+                                                             server.uuid)
+        except Exception as e:
+            with excutils.save_and_reraise_exception():
+                LOG.exception("Set server power state to %(state) failed, the "
+                              "reason: %(reason)s",
+                              {"state": state, "reason": six.text_type(e)})
+                server.power_state = self.driver.get_power_state(context,
+                                                                 server.uuid)
+                if state in ['reboot', 'soft_reboot'] \
+                        and server.power_state != states.POWER_ON:
+                    utils.process_event(fsm, server, event='error')
+                else:
+                    utils.process_event(fsm, server, event='fail')
+
+                action = POWER_NOTIFICATION_MAP[state]
+                notifications.notify_about_server_action(
+                    context, server, self.host,
+                    action=action,
+                    phase=fields.NotificationPhase.ERROR,
+                    exception=e)
+
         utils.process_event(fsm, server, event='done')
         LOG.info('Successfully set node power state: %s',
                  state, server=server)

mogan/objects/fields.py

@@ -145,7 +145,9 @@ class NotificationAction(BaseMoganEnum):
     DELETE = 'delete'
     POWER_ON = 'power_on'
     POWER_OFF = 'power_off'
+    SOFT_POWER_OFF = 'soft_off'
     REBOOT = 'reboot'
+    SOFT_REBOOT = 'soft_reboot'
     SHUTDOWN = 'shutdown'
     CREATE = 'create'
 

mogan/tests/unit/engine/test_manager.py

@@ -15,6 +15,7 @@
 
 """Test class for Mogan ManagerService."""
 
+from ironicclient import exc as ironic_exc
 import mock
 from oslo_config import cfg
 from oslo_utils import uuidutils
@@ -26,6 +27,8 @@ from mogan.common import ironic
 from mogan.common import states
 from mogan.engine import manager
 from mogan.network import api as network_api
+from mogan.notifications import base as notifications
+from mogan.objects import fields
 from mogan.tests.unit.db import base as tests_db_base
 from mogan.tests.unit.engine import mgr_utils
 from mogan.tests.unit.objects import utils as obj_utils
@@ -123,6 +126,54 @@ class ManageServerTestCase(mgr_utils.ServiceSetUpMixin,
                                                ironic_states.POWER_ON)
         get_power_mock.assert_called_once_with(self.context, server.uuid)
 
+    @mock.patch.object(notifications, 'notify_about_server_action')
+    @mock.patch.object(IronicDriver, 'get_power_state')
+    @mock.patch.object(IronicDriver, 'set_power_state')
+    def test_change_server_power_state_with_error_status(
+            self, set_power_mock, get_power_mock, notify_mock):
+        server = obj_utils.create_test_server(
+            self.context, status=states.REBOOTING)
+        get_power_mock.return_value = states.POWER_OFF
+        exception = ironic_exc.NotFound("The bare metal node is not found")
+        set_power_mock.side_effect = exception
+
+        self._start_service()
+        self.assertRaises(ironic_exc.NotFound,
+                          self.service.set_power_state,
+                          self.context,
+                          server, 'reboot')
+        set_power_mock.assert_called_once_with(self.context, server, 'reboot')
+        get_power_mock.assert_called_once_with(self.context, server.uuid)
+        notify_mock.assert_called_once_with(
+            self.context, server, 'test-host',
+            action=fields.NotificationAction.REBOOT,
+            phase=fields.NotificationPhase.ERROR, exception=exception)
+        self.assertEqual(server.status, states.ERROR)
+        self._stop_service()
+
+    @mock.patch.object(notifications, 'notify_about_server_action')
+    @mock.patch.object(IronicDriver, 'get_power_state')
+    @mock.patch.object(IronicDriver, 'set_power_state')
+    def test_change_server_power_state_with_rollback_status(
+            self, set_power_mock, get_power_mock, notify_mock):
+        server = obj_utils.create_test_server(
+            self.context, status=states.POWERING_OFF)
+        exception = ironic_exc.NotFound("The bare metal node is not found")
+        get_power_mock.return_value = states.POWER_ON
+        set_power_mock.side_effect = exception
+
+        self._start_service()
+        self.assertRaises(ironic_exc.NotFound, self.service.set_power_state,
+                          self.context, server, 'off')
+        set_power_mock.assert_called_once_with(self.context, server, 'off')
+        get_power_mock.assert_called_once_with(self.context, server.uuid)
+        notify_mock.assert_called_once_with(
+            self.context, server, 'test-host',
+            action=fields.NotificationAction.POWER_OFF,
+            phase=fields.NotificationPhase.ERROR, exception=exception)
+        self.assertEqual(server.status, states.ACTIVE)
+        self._stop_service()
+
     @mock.patch.object(ironic.IronicClientWrapper, 'call')
     def test_get_serial_console(self, mock_ironic_call):
         fake_node = mock.MagicMock()