Introduce a state machine for instance status

State Machine helps us to manage instance states transitions.

Change-Id: Ie7b7d33ea509484a98344e0a95728075d6aebe47
This commit is contained in:
Zhenguo Niu 2017-01-12 00:25:09 +08:00
parent 278a2eb8cf
commit b28a51c904
12 changed files with 459 additions and 62 deletions

View File

@ -257,4 +257,12 @@ class AZNotFound(NotFound):
msg_fmt = _("The availability zone could not be found.")
class InvalidState(Invalid):
_msg_fmt = _("Invalid resource state.")
class DuplicateState(Conflict):
_msg_fmt = _("Resource already exists.")
ObjectActionError = obj_exc.ObjectActionError

151
mogan/common/fsm.py Normal file
View File

@ -0,0 +1,151 @@
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from automaton import exceptions as automaton_exceptions
from automaton import machines
import six
from mogan.common import exception as excp
from mogan.common.i18n import _
"""State machine modelling."""
def _translate_excp(func):
"""Decorator to translate automaton exceptions into mogan exceptions."""
@six.wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except (automaton_exceptions.InvalidState,
automaton_exceptions.NotInitialized,
automaton_exceptions.FrozenMachine,
automaton_exceptions.NotFound) as e:
raise excp.InvalidState(six.text_type(e))
except automaton_exceptions.Duplicate as e:
raise excp.DuplicateState(six.text_type(e))
return wrapper
class FSM(machines.FiniteMachine):
"""An mogan state-machine class with some mogan specific additions."""
def __init__(self):
super(FSM, self).__init__()
self._target_state = None
# For now make these raise mogan state machine exceptions until
# a later period where these should(?) be using the raised automaton
# exceptions directly.
add_transition = _translate_excp(machines.FiniteMachine.add_transition)
@property
def target_state(self):
return self._target_state
def is_stable(self, state):
"""Is the state stable?
:param state: the state of interest
:raises: InvalidState if the state is invalid
:returns: True if it is a stable state; False otherwise
"""
try:
return self._states[state]['stable']
except KeyError:
raise excp.InvalidState(_("State '%s' does not exist") % state)
@_translate_excp
def add_state(self, state, on_enter=None, on_exit=None,
target=None, terminal=None, stable=False):
"""Adds a given state to the state machine.
:param stable: Use this to specify that this state is a stable/passive
state. A state must have been previously defined as
'stable' before it can be used as a 'target'
:param target: The target state for 'state' to go to. Before a state
can be used as a target it must have been previously
added and specified as 'stable'
Further arguments are interpreted as for parent method ``add_state``.
"""
self._validate_target_state(target)
super(FSM, self).add_state(state, terminal=terminal,
on_enter=on_enter, on_exit=on_exit)
self._states[state].update({
'stable': stable,
'target': target,
})
def _post_process_event(self, event, result):
# Clear '_target_state' if we've reached it
if (self._target_state is not None and
self._target_state == self._current.name):
self._target_state = None
# If new state has a different target, update the '_target_state'
if self._states[self._current.name]['target'] is not None:
self._target_state = self._states[self._current.name]['target']
def _validate_target_state(self, target):
"""Validate the target state.
A target state must be a valid state that is 'stable'.
:param target: The target state
:raises: exception.InvalidState if it is an invalid target state
"""
if target is None:
return
if target not in self._states:
raise excp.InvalidState(
_("Target state '%s' does not exist") % target)
if not self.is_stable(target):
raise excp.InvalidState(
_("Target state '%s' is not a 'stable' state") % target)
@_translate_excp
def initialize(self, start_state=None, target_state=None):
"""Initialize the FSM.
:param start_state: the FSM is initialized to start from this state
:param target_state: if specified, the FSM is initialized to this
target state. Otherwise use the default target
state
"""
super(FSM, self).initialize(start_state=start_state)
current_state = self._current.name
self._validate_target_state(target_state)
self._target_state = (target_state or
self._states[current_state]['target'])
@_translate_excp
def process_event(self, event, target_state=None):
"""process the event.
:param event: the event to be processed
:param target_state: if specified, the 'final' target state for the
event. Otherwise, use the default target state
"""
super(FSM, self).process_event(event)
if target_state:
# NOTE(rloo): _post_process_event() was invoked at the end of
# the above super().process_event() call. At this
# point, the default target state is being used but
# we want to use the specified state instead.
self._validate_target_state(target_state)
self._target_state = target_state

149
mogan/common/states.py Normal file
View File

@ -0,0 +1,149 @@
# Copyright 2016 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Mapping of bare metal instance states.
Setting the instance `power_state` is handled by the engine's power
synchronization thread. Based on the power state retrieved from the
hypervisor for the instance.
"""
from oslo_log import log as logging
from mogan.common import fsm
LOG = logging.getLogger(__name__)
#################
# Instance states
#################
""" Mapping of state-changing events that are PUT to the REST API
This is a mapping of target states which are PUT to the API.
This provides a reference set of supported actions, and in the future
may be used to support renaming these actions.
"""
ACTIVE = 'active'
""" The server is active """
BUILDING = 'building'
""" The server has not finished the original build process """
DELETED = 'deleted'
""" The server is permanently deleted """
DELETING = 'deleting'
""" The server has not finished the original delete process """
ERROR = 'error'
""" The server is in error """
POWERING_ON = 'powering-on'
""" The server is in powering on """
POWERING_OFF = 'powering-off'
""" The server is in powering off """
REBOOTING = 'rebooting'
""" The server is in rebooting """
STOPPED = 'stopped'
""" The server is powered off """
REBUILDING = 'rebuilding'
""" The server is in rebuilding process """
STABLE_STATES = (ACTIVE, ERROR, DELETED, STOPPED)
"""States that will not transition unless receiving a request."""
UNSTABLE_STATES = (BUILDING, DELETING, POWERING_ON, POWERING_OFF, REBOOTING,
REBUILDING)
"""States that can be changed without external request."""
#####################
# State machine model
#####################
def on_exit(old_state, event):
"""Used to log when a state is exited."""
LOG.debug("Exiting old state '%s' in response to event '%s'",
old_state, event)
def on_enter(new_state, event):
"""Used to log when entering a state."""
LOG.debug("Entering new state '%s' in response to event '%s'",
new_state, event)
watchers = {}
watchers['on_exit'] = on_exit
watchers['on_enter'] = on_enter
machine = fsm.FSM()
# Add stable states
for state in STABLE_STATES:
machine.add_state(state, stable=True, **watchers)
# Add build* states
machine.add_state(BUILDING, target=ACTIVE, **watchers)
# Add delete* states
machine.add_state(DELETING, target=DELETED, **watchers)
# Add rebuild* states
machine.add_state(REBUILDING, target=ACTIVE, **watchers)
# Add power on* states
machine.add_state(POWERING_ON, target=ACTIVE, **watchers)
# Add power off* states
machine.add_state(POWERING_OFF, target=STOPPED, **watchers)
# Add reboot* states
machine.add_state(REBOOTING, target=ACTIVE, **watchers)
# from active* states
machine.add_transition(ACTIVE, REBUILDING, 'rebuild')
machine.add_transition(ACTIVE, POWERING_OFF, 'stop')
machine.add_transition(ACTIVE, REBOOTING, 'reboot')
machine.add_transition(ACTIVE, DELETING, 'delete')
# from stopped* states
machine.add_transition(STOPPED, POWERING_ON, 'start')
machine.add_transition(STOPPED, REBUILDING, 'rebuild')
machine.add_transition(STOPPED, DELETING, 'delete')
# from error* states
machine.add_transition(ERROR, DELETING, 'delete')
# from *ing states
machine.add_transition(BUILDING, ACTIVE, 'done')
machine.add_transition(DELETING, DELETED, 'done')
machine.add_transition(REBUILDING, ACTIVE, 'done')
machine.add_transition(POWERING_ON, ACTIVE, 'done')
machine.add_transition(POWERING_OFF, STOPPED, 'done')
machine.add_transition(REBOOTING, ACTIVE, 'done')
# All unstable states are allowed to transition to ERROR and DELETING
for state in UNSTABLE_STATES:
machine.add_transition(state, ERROR, 'error')
machine.add_transition(state, DELETING, 'delete')

View File

@ -18,9 +18,9 @@
from oslo_log import log
from mogan.common import exception
from mogan.common import states
from mogan.conf import CONF
from mogan.engine import rpcapi
from mogan.engine import status
from mogan import image
from mogan import objects
@ -45,7 +45,7 @@ class API(object):
base_options = {
'image_uuid': image_uuid,
'status': status.BUILDING,
'status': states.BUILDING,
'user_id': context.user,
'project_id': context.tenant,
'instance_type_uuid': instance_type['uuid'],
@ -62,7 +62,7 @@ class API(object):
instance = objects.Instance(context=context)
instance.update(base_options)
instance.status = status.BUILDING
instance.status = states.BUILDING
instance.create()
return instance
@ -123,8 +123,14 @@ class API(object):
requested_networks)
def _delete_instance(self, context, instance):
# Initialize state machine
fsm = states.machine.copy()
fsm.initialize(start_state=instance.status,
target_state=states.DELETED)
fsm.process_event('delete')
try:
instance.status = status.DELETING
instance.status = fsm.current_state
instance.save()
except exception.InstanceNotFound:
LOG.debug("Instance %s is not found while deleting",

View File

@ -18,7 +18,6 @@ import traceback
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import timeutils
import taskflow.engines
from taskflow.patterns import linear_flow
@ -27,10 +26,10 @@ from mogan.common import flow_utils
from mogan.common.i18n import _
from mogan.common.i18n import _LE
from mogan.common.i18n import _LI
from mogan.common import states
from mogan.common import utils
from mogan.engine.baremetal import ironic
from mogan.engine.baremetal import ironic_states
from mogan.engine import status
LOG = logging.getLogger(__name__)
@ -273,7 +272,7 @@ class CreateInstanceTask(flow_utils.MoganTask):
def _wait_for_active(self, instance):
"""Wait for the node to be marked as ACTIVE in Ironic."""
instance.refresh()
if instance.status in (status.DELETING, status.ERROR, status.DELETED):
if instance.status in (states.DELETING, states.ERROR, states.DELETED):
raise exception.InstanceDeployFailure(
_("Instance %s provisioning was aborted") % instance.uuid)
@ -284,9 +283,6 @@ class CreateInstanceTask(flow_utils.MoganTask):
# job is done
LOG.debug("Ironic node %(node)s is now ACTIVE",
dict(node=node.uuid))
instance.status = status.ACTIVE
instance.launched_at = timeutils.utcnow()
instance.save()
raise loopingcall.LoopingCallDone()
if node.target_provision_state in (ironic_states.DELETED,

View File

@ -29,12 +29,12 @@ from mogan.common.i18n import _
from mogan.common.i18n import _LE
from mogan.common.i18n import _LI
from mogan.common.i18n import _LW
from mogan.common import states
from mogan.conf import CONF
from mogan.engine.baremetal import ironic
from mogan.engine.baremetal import ironic_states
from mogan.engine import base_manager
from mogan.engine.flows import create_instance
from mogan.engine import status
from mogan.notifications import base as notifications
from mogan.objects import fields
@ -71,14 +71,6 @@ class EngineManager(base_manager.BaseEngineManager):
def _sync_node_resources(self, context):
self._refresh_cache()
def _set_instance_obj_error_state(self, context, instance):
try:
instance.status = status.ERROR
instance.save()
except exception.InstanceNotFound:
LOG.debug('Instance has been destroyed from under us while '
'trying to set it to ERROR', instance=instance)
def destroy_networks(self, context, instance):
LOG.debug("unplug: instance_uuid=%(uuid)s vif=%(network_info)s",
{'uuid': instance.uuid,
@ -170,6 +162,10 @@ class EngineManager(base_manager.BaseEngineManager):
action=fields.NotificationAction.CREATE,
phase=fields.NotificationPhase.START)
# Initialize state machine
fsm = states.machine.copy()
fsm.initialize(start_state=instance.status, target_state=states.ACTIVE)
if filter_properties is None:
filter_properties = {}
@ -198,12 +194,21 @@ class EngineManager(base_manager.BaseEngineManager):
try:
_run_flow()
except Exception as e:
self._set_instance_obj_error_state(context, instance)
fsm.process_event('error')
instance.status = fsm.current_state
instance.save()
LOG.error(_LE("Created instance %(uuid)s failed."
"Exception: %(exception)s"),
{"uuid": instance.uuid,
"exception": e})
else:
# Advance the state model for the given event. Note that this
# doesn't alter the instance in any way. This may raise
# InvalidState, if this event is not allowed in the current state.
fsm.process_event('done')
instance.status = fsm.current_state
instance.launched_at = timeutils.utcnow()
instance.save()
LOG.info(_LI("Created instance %s successfully."), instance.uuid)
finally:
return instance
@ -212,6 +217,11 @@ class EngineManager(base_manager.BaseEngineManager):
"""Delete an instance."""
LOG.debug("Deleting instance...")
# Initialize state machine
fsm = states.machine.copy()
fsm.initialize(start_state=instance.status,
target_state=states.DELETED)
try:
node = ironic.get_node_by_instance(self.ironicclient,
instance.uuid)
@ -229,8 +239,13 @@ class EngineManager(base_manager.BaseEngineManager):
LOG.exception(_LE("Error while trying to clean up "
"instance resources."),
instance=instance)
fsm.process_event('error')
instance.status = fsm.current_state
instance.save()
return
instance.status = status.DELETED
fsm.process_event('done')
instance.status = fsm.current_state
instance.deleted_at = timeutils.utcnow()
instance.save()
instance.destroy()

View File

@ -1,32 +0,0 @@
# Copyright 2016 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Possible status for instances.
Compute instance status represent the state of an instance as it pertains to
a user or administrator.
"""
# Instance is running
ACTIVE = 'active'
# Instance only exists in DB
BUILDING = 'building'
DELETING = 'deleting'
DELETED = 'deleted'
ERROR = 'error'

View File

@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from mogan.common import exception as excp
from mogan.common import fsm
from mogan.tests import base
class FSMTest(base.TestCase):
def setUp(self):
super(FSMTest, self).setUp()
m = fsm.FSM()
m.add_state('working', stable=True)
m.add_state('daydream')
m.add_state('wakeup', target='working')
m.add_state('play', stable=True)
m.add_transition('wakeup', 'working', 'walk')
self.fsm = m
def test_is_stable(self):
self.assertTrue(self.fsm.is_stable('working'))
def test_is_stable_not(self):
self.assertFalse(self.fsm.is_stable('daydream'))
def test_is_stable_invalid_state(self):
self.assertRaises(excp.InvalidState, self.fsm.is_stable, 'foo')
def test_target_state_stable(self):
# Test to verify that adding a new state with a 'target' state pointing
# to a 'stable' state does not raise an exception
self.fsm.add_state('foo', target='working')
self.fsm.default_start_state = 'working'
self.fsm.initialize()
def test__validate_target_state(self):
# valid
self.fsm._validate_target_state('working')
# target doesn't exist
self.assertRaisesRegex(excp.InvalidState, "does not exist",
self.fsm._validate_target_state, 'new state')
# target isn't a stable state
self.assertRaisesRegex(excp.InvalidState, "stable",
self.fsm._validate_target_state, 'daydream')
def test_initialize(self):
# no start state
self.assertRaises(excp.InvalidState, self.fsm.initialize)
# no target state
self.fsm.initialize('working')
self.assertEqual('working', self.fsm.current_state)
self.assertIsNone(self.fsm.target_state)
# default target state
self.fsm.initialize('wakeup')
self.assertEqual('wakeup', self.fsm.current_state)
self.assertEqual('working', self.fsm.target_state)
# specify (it overrides default) target state
self.fsm.initialize('wakeup', 'play')
self.assertEqual('wakeup', self.fsm.current_state)
self.assertEqual('play', self.fsm.target_state)
# specify an invalid target state
self.assertRaises(excp.InvalidState, self.fsm.initialize,
'wakeup', 'daydream')
def test_process_event(self):
# default target state
self.fsm.initialize('wakeup')
self.fsm.process_event('walk')
self.assertEqual('working', self.fsm.current_state)
self.assertIsNone(self.fsm.target_state)
# specify (it overrides default) target state
self.fsm.initialize('wakeup')
self.fsm.process_event('walk', 'play')
self.assertEqual('working', self.fsm.current_state)
self.assertEqual('play', self.fsm.target_state)
# specify an invalid target state
self.fsm.initialize('wakeup')
self.assertRaises(excp.InvalidState, self.fsm.process_event,
'walk', 'daydream')

View File

@ -16,8 +16,8 @@
from oslo_utils import uuidutils
from mogan.common import states
from mogan.db import api as db_api
from mogan.engine import status
def get_test_instance(**kw):
@ -46,7 +46,7 @@ def get_test_instance(**kw):
'project_id': kw.get('project_id',
'c18e8a1a870d4c08a0b51ced6e0b6459'),
'user_id': kw.get('user_id', 'cdbf77d47f1d4d04ad9b7ff62b672467'),
'status': kw.get('status', status.ACTIVE),
'status': kw.get('status', states.ACTIVE),
'instance_type_uuid': kw.get('instance_type_uuid',
'28708dff-283c-449e-9bfa-a48c93480c86'),
'availability_zone': kw.get('availability_zone', 'test_az'),

View File

@ -20,9 +20,9 @@ from oslo_config import cfg
from oslo_context import context
from mogan.common import exception
from mogan.common import states
from mogan.engine import api as engine_api
from mogan.engine import rpcapi as engine_rpcapi
from mogan.engine import status
from mogan import objects
from mogan.tests.unit.db import base
from mogan.tests.unit.db import utils as db_utils
@ -58,7 +58,7 @@ class ComputeAPIUnitTest(base.DbTestCase):
self.assertEqual('fake-user', base_opts['user_id'])
self.assertEqual('fake-project', base_opts['project_id'])
self.assertEqual(status.BUILDING, base_opts['status'])
self.assertEqual(states.BUILDING, base_opts['status'])
self.assertEqual(instance_type.uuid, base_opts['instance_type_uuid'])
self.assertEqual({'k1', 'v1'}, base_opts['extra'])
self.assertEqual('test_az', base_opts['availability_zone'])
@ -68,7 +68,7 @@ class ComputeAPIUnitTest(base.DbTestCase):
mock_inst_create.return_value = mock.MagicMock()
base_options = {'image_uuid': 'fake-uuid',
'status': status.BUILDING,
'status': states.BUILDING,
'user_id': 'fake-user',
'project_id': 'fake-project',
'instance_type_uuid': 'fake-type-uuid',
@ -90,7 +90,7 @@ class ComputeAPIUnitTest(base.DbTestCase):
instance_type = self._create_instance_type()
base_options = {'image_uuid': 'fake-uuid',
'status': status.BUILDING,
'status': states.BUILDING,
'user_id': 'fake-user',
'project_id': 'fake-project',
'instance_type_uuid': 'fake-type-uuid',
@ -132,7 +132,7 @@ class ComputeAPIUnitTest(base.DbTestCase):
instance_type = self._create_instance_type()
base_options = {'image_uuid': 'fake-uuid',
'status': status.BUILDING,
'status': states.BUILDING,
'user_id': 'fake-user',
'project_id': 'fake-project',
'instance_type_uuid': 'fake-type-uuid',

View File

@ -19,6 +19,7 @@ import mock
from oslo_config import cfg
from mogan.common import exception
from mogan.common import states
from mogan.engine.baremetal import ironic
from mogan.engine.baremetal import ironic_states
from mogan.engine import manager
@ -117,7 +118,8 @@ class ManageInstanceTestCase(mgr_utils.ServiceSetUpMixin,
refresh_cache_mock):
fake_node = mock.MagicMock()
fake_node.provision_state = ironic_states.ACTIVE
instance = obj_utils.create_test_instance(self.context)
instance = obj_utils.create_test_instance(
self.context, status=states.DELETING)
destroy_net_mock.side_effect = None
destroy_inst_mock.side_effect = None
refresh_cache_mock.side_effect = None
@ -137,7 +139,8 @@ class ManageInstanceTestCase(mgr_utils.ServiceSetUpMixin,
self, destroy_inst_mock, get_node_mock, refresh_cache_mock):
fake_node = mock.MagicMock()
fake_node.provision_state = 'foo'
instance = obj_utils.create_test_instance(self.context)
instance = obj_utils.create_test_instance(
self.context, status=states.DELETING)
destroy_inst_mock.side_effect = None
refresh_cache_mock.side_effect = None
get_node_mock.return_value = fake_node

View File

@ -30,3 +30,4 @@ taskflow>=2.7.0 # Apache-2.0
WSME>=0.8 # MIT
keystonemiddleware>=4.12.0 # Apache-2.0
stevedore>=1.17.1 # Apache-2.0
automaton>=0.5.0 # Apache-2.0