Rollback to the original status when server powering action failed
As we don't move server status to error when power action failed, the status will be stuck in *ing forever. Change-Id: Ie98ec4c5d2d848ca13dae4d891f762cd58650fad Closes-Bug: #1690096
This commit is contained in:
parent
30a16bc608
commit
4b80551346
|
@ -195,6 +195,12 @@ machine.add_transition(SOFT_POWERING_OFF, STOPPED, 'done')
|
||||||
machine.add_transition(REBOOTING, ACTIVE, 'done')
|
machine.add_transition(REBOOTING, ACTIVE, 'done')
|
||||||
machine.add_transition(SOFT_REBOOTING, ACTIVE, 'done')
|
machine.add_transition(SOFT_REBOOTING, ACTIVE, 'done')
|
||||||
|
|
||||||
|
machine.add_transition(POWERING_ON, STOPPED, 'fail')
|
||||||
|
machine.add_transition(POWERING_OFF, ACTIVE, 'fail')
|
||||||
|
machine.add_transition(SOFT_POWERING_OFF, ACTIVE, 'fail')
|
||||||
|
machine.add_transition(REBOOTING, ACTIVE, 'fail')
|
||||||
|
machine.add_transition(SOFT_REBOOTING, ACTIVE, 'fail')
|
||||||
|
|
||||||
# All unstable states are allowed to transition to ERROR and DELETING
|
# All unstable states are allowed to transition to ERROR and DELETING
|
||||||
for state in UNSTABLE_STATES:
|
for state in UNSTABLE_STATES:
|
||||||
machine.add_transition(state, ERROR, 'error')
|
machine.add_transition(state, ERROR, 'error')
|
||||||
|
|
|
@ -42,6 +42,14 @@ from mogan.scheduler import utils as sched_utils
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
POWER_NOTIFICATION_MAP = {
|
||||||
|
'on': fields.NotificationAction.POWER_ON,
|
||||||
|
'off': fields.NotificationAction.POWER_OFF,
|
||||||
|
'reboot': fields.NotificationAction.REBOOT,
|
||||||
|
'soft_off': fields.NotificationAction.SOFT_POWER_OFF,
|
||||||
|
'soft_reboot': fields.NotificationAction.SOFT_REBOOT
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@utils.expects_func_args('server')
|
@utils.expects_func_args('server')
|
||||||
def wrap_server_fault(function):
|
def wrap_server_fault(function):
|
||||||
|
@ -467,6 +475,7 @@ class EngineManager(base_manager.BaseEngineManager):
|
||||||
server.destroy()
|
server.destroy()
|
||||||
LOG.info("Deleted server successfully.")
|
LOG.info("Deleted server successfully.")
|
||||||
|
|
||||||
|
@wrap_server_fault
|
||||||
def set_power_state(self, context, server, state):
|
def set_power_state(self, context, server, state):
|
||||||
"""Set power state for the specified server."""
|
"""Set power state for the specified server."""
|
||||||
|
|
||||||
|
@ -479,9 +488,30 @@ class EngineManager(base_manager.BaseEngineManager):
|
||||||
'server': server})
|
'server': server})
|
||||||
self.driver.set_power_state(context, server, state)
|
self.driver.set_power_state(context, server, state)
|
||||||
|
|
||||||
|
try:
|
||||||
do_set_power_state()
|
do_set_power_state()
|
||||||
server.power_state = self.driver.get_power_state(context,
|
server.power_state = self.driver.get_power_state(context,
|
||||||
server.uuid)
|
server.uuid)
|
||||||
|
except Exception as e:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
LOG.exception("Set server power state to %(state) failed, the "
|
||||||
|
"reason: %(reason)s",
|
||||||
|
{"state": state, "reason": six.text_type(e)})
|
||||||
|
server.power_state = self.driver.get_power_state(context,
|
||||||
|
server.uuid)
|
||||||
|
if state in ['reboot', 'soft_reboot'] \
|
||||||
|
and server.power_state != states.POWER_ON:
|
||||||
|
utils.process_event(fsm, server, event='error')
|
||||||
|
else:
|
||||||
|
utils.process_event(fsm, server, event='fail')
|
||||||
|
|
||||||
|
action = POWER_NOTIFICATION_MAP[state]
|
||||||
|
notifications.notify_about_server_action(
|
||||||
|
context, server, self.host,
|
||||||
|
action=action,
|
||||||
|
phase=fields.NotificationPhase.ERROR,
|
||||||
|
exception=e)
|
||||||
|
|
||||||
utils.process_event(fsm, server, event='done')
|
utils.process_event(fsm, server, event='done')
|
||||||
LOG.info('Successfully set node power state: %s',
|
LOG.info('Successfully set node power state: %s',
|
||||||
state, server=server)
|
state, server=server)
|
||||||
|
|
|
@ -145,7 +145,9 @@ class NotificationAction(BaseMoganEnum):
|
||||||
DELETE = 'delete'
|
DELETE = 'delete'
|
||||||
POWER_ON = 'power_on'
|
POWER_ON = 'power_on'
|
||||||
POWER_OFF = 'power_off'
|
POWER_OFF = 'power_off'
|
||||||
|
SOFT_POWER_OFF = 'soft_off'
|
||||||
REBOOT = 'reboot'
|
REBOOT = 'reboot'
|
||||||
|
SOFT_REBOOT = 'soft_reboot'
|
||||||
SHUTDOWN = 'shutdown'
|
SHUTDOWN = 'shutdown'
|
||||||
CREATE = 'create'
|
CREATE = 'create'
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
|
|
||||||
"""Test class for Mogan ManagerService."""
|
"""Test class for Mogan ManagerService."""
|
||||||
|
|
||||||
|
from ironicclient import exc as ironic_exc
|
||||||
import mock
|
import mock
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
|
@ -26,6 +27,8 @@ from mogan.common import ironic
|
||||||
from mogan.common import states
|
from mogan.common import states
|
||||||
from mogan.engine import manager
|
from mogan.engine import manager
|
||||||
from mogan.network import api as network_api
|
from mogan.network import api as network_api
|
||||||
|
from mogan.notifications import base as notifications
|
||||||
|
from mogan.objects import fields
|
||||||
from mogan.tests.unit.db import base as tests_db_base
|
from mogan.tests.unit.db import base as tests_db_base
|
||||||
from mogan.tests.unit.engine import mgr_utils
|
from mogan.tests.unit.engine import mgr_utils
|
||||||
from mogan.tests.unit.objects import utils as obj_utils
|
from mogan.tests.unit.objects import utils as obj_utils
|
||||||
|
@ -123,6 +126,54 @@ class ManageServerTestCase(mgr_utils.ServiceSetUpMixin,
|
||||||
ironic_states.POWER_ON)
|
ironic_states.POWER_ON)
|
||||||
get_power_mock.assert_called_once_with(self.context, server.uuid)
|
get_power_mock.assert_called_once_with(self.context, server.uuid)
|
||||||
|
|
||||||
|
@mock.patch.object(notifications, 'notify_about_server_action')
|
||||||
|
@mock.patch.object(IronicDriver, 'get_power_state')
|
||||||
|
@mock.patch.object(IronicDriver, 'set_power_state')
|
||||||
|
def test_change_server_power_state_with_error_status(
|
||||||
|
self, set_power_mock, get_power_mock, notify_mock):
|
||||||
|
server = obj_utils.create_test_server(
|
||||||
|
self.context, status=states.REBOOTING)
|
||||||
|
get_power_mock.return_value = states.POWER_OFF
|
||||||
|
exception = ironic_exc.NotFound("The bare metal node is not found")
|
||||||
|
set_power_mock.side_effect = exception
|
||||||
|
|
||||||
|
self._start_service()
|
||||||
|
self.assertRaises(ironic_exc.NotFound,
|
||||||
|
self.service.set_power_state,
|
||||||
|
self.context,
|
||||||
|
server, 'reboot')
|
||||||
|
set_power_mock.assert_called_once_with(self.context, server, 'reboot')
|
||||||
|
get_power_mock.assert_called_once_with(self.context, server.uuid)
|
||||||
|
notify_mock.assert_called_once_with(
|
||||||
|
self.context, server, 'test-host',
|
||||||
|
action=fields.NotificationAction.REBOOT,
|
||||||
|
phase=fields.NotificationPhase.ERROR, exception=exception)
|
||||||
|
self.assertEqual(server.status, states.ERROR)
|
||||||
|
self._stop_service()
|
||||||
|
|
||||||
|
@mock.patch.object(notifications, 'notify_about_server_action')
|
||||||
|
@mock.patch.object(IronicDriver, 'get_power_state')
|
||||||
|
@mock.patch.object(IronicDriver, 'set_power_state')
|
||||||
|
def test_change_server_power_state_with_rollback_status(
|
||||||
|
self, set_power_mock, get_power_mock, notify_mock):
|
||||||
|
server = obj_utils.create_test_server(
|
||||||
|
self.context, status=states.POWERING_OFF)
|
||||||
|
exception = ironic_exc.NotFound("The bare metal node is not found")
|
||||||
|
get_power_mock.return_value = states.POWER_ON
|
||||||
|
set_power_mock.side_effect = exception
|
||||||
|
|
||||||
|
self._start_service()
|
||||||
|
self.assertRaises(ironic_exc.NotFound, self.service.set_power_state,
|
||||||
|
self.context, server, 'off')
|
||||||
|
set_power_mock.assert_called_once_with(self.context, server, 'off')
|
||||||
|
get_power_mock.assert_called_once_with(self.context, server.uuid)
|
||||||
|
notify_mock.assert_called_once_with(
|
||||||
|
self.context, server, 'test-host',
|
||||||
|
action=fields.NotificationAction.POWER_OFF,
|
||||||
|
phase=fields.NotificationPhase.ERROR, exception=exception)
|
||||||
|
self.assertEqual(server.status, states.ACTIVE)
|
||||||
|
self._stop_service()
|
||||||
|
|
||||||
@mock.patch.object(ironic.IronicClientWrapper, 'call')
|
@mock.patch.object(ironic.IronicClientWrapper, 'call')
|
||||||
def test_get_serial_console(self, mock_ironic_call):
|
def test_get_serial_console(self, mock_ironic_call):
|
||||||
fake_node = mock.MagicMock()
|
fake_node = mock.MagicMock()
|
||||||
|
|
Loading…
Reference in New Issue