openstack/masakari
Code Issues Proposed changes

Add hooks for recovery method customization plugins

Browse Source 
This change adds hooks for pulling in plugins for
Interoperable recovery method customization taksflow
by configuring the actions in terms of execution order,
extra parameters to execute commands in action.

Implements: blueprint recovery-method-customization

Change-Id: I8150935ce0a4e592bd0f87bb3c7b3e599cefa033
This commit is contained in:
bhagyashris 2018-02-05 16:38:02 +05:30 committed by Pooja Jadhav
parent cf0d30b8c4
commit ad3dc737c9
16 changed files with 598 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
doc/source/_static/masakari-custom-recovery-methods.conf.sample Normal file
View File

@ -0,0 +1,87 @@
[DEFAULT]
[taskflow_driver_recovery_flows]
#
# From customized_recovery_flow_opts
#
#
# This option allows operator to customize tasks to be executed for host failure
# auto recovery workflow.
#
# Provide list of strings reflecting to the task classes that should be included
# to the host failure recovery workflow. The full classname path of all task
# classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
# these classes may be implemented by OpenStack Masaskari project team, deployer
# or third party.
#
# By default below three tasks will be part of this config option:-
# 1. disable_compute_service_task
# 2. prepare_HA_enabled_instances_task
# 3. evacuate_instances_task
#
# The allowed values for this option is comma separated dictionary of object
# names in between ``{`` and ``}``. (dict value)
#host_auto_failure_recovery_tasks = main:['prepare_HA_enabled_instances_task'],
#post:['evacuate_instances_task'],pre:['disable_compute_service_task']
#
# This option allows operator to customize tasks to be executed for host failure
# reserved_host recovery workflow.
#
# Provide list of strings reflecting to the task classes that should be included
# to the host failure recovery workflow. The full classname path of all task
# classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
# these classes may be implemented by OpenStack Masaskari project team, deployer
# or third party.
#
# By default below three tasks will be part of this config option:-
# 1. disable_compute_service_task
# 2. prepare_HA_enabled_instances_task
# 3. evacuate_instances_task
#
# The allowed values for this option is comma separated dictionary of object
# names in between ``{`` and ``}``. (dict value)
#host_rh_failure_recovery_tasks = main:['prepare_HA_enabled_instances_task',
#'evacuate_instances_task'],post:[],pre:['disable_compute_service_task']
#
# This option allows operator to customize tasks to be executed for instance
# failure recovery workflow.
#
# Provide list of strings reflecting to the task classes that should be included
# to the instance failure recovery workflow. The full classname path of all task
# classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
# these classes may be implemented by OpenStack Masaskari project team, deployer
# or third party.
#
# By default below three tasks will be part of this config option:-
# 1. stop_instance_task
# 2. start_instance_task
# 3. confirm_instance_active_task
#
# The allowed values for this option is comma separated dictionary of object
# names in between ``{`` and ``}``. (dict value)
#instance_failure_recovery_tasks = main:['start_instance_task'],
#post:['confirm_instance_active_task'],pre:['stop_instance_task']
#
# This option allows operator to customize tasks to be executed for process
# failure recovery workflow.
#
# Provide list of strings reflecting to the task classes that should be included
# to the process failure recovery workflow. The full classname path of all task
# classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
# these classes may be implemented by OpenStack Masaskari project team, deployer
# or third party.
#
# By default below two tasks will be part of this config option:-
# 1. disable_compute_node_task
# 2. confirm_compute_node_disabled_task
#
# The allowed values for this option is comma separated dictionary of object
# names in between ``{`` and ``}``. (dict value)
#process_failure_recovery_tasks = main:['confirm_compute_node_disabled_task'],
#post:[],pre:['disable_compute_node_task']

2
doc/source/index.rst
View File

@ -68,6 +68,8 @@ This section will help you in configuring masakari mannualy.
operators_guide
sample_config
sample_policy
recovery_workflow_sample_config
recovery_workflow_custom_task
Indices and tables
==================

65
doc/source/recovery_workflow_custom_task.rst Normal file
View File

@ -0,0 +1,65 @@
================================================
Guide for Custom Recovery Workflow Configuration
================================================
If operator wants customized recovery workflow, so here is guidelines mentioned
for how to associate custom tasks from Third Party Library along with standard
recovery workflows in Masakari.:
#. First make sure required Third Party Library is installed on the Masakari
engine node. Below is the sample custom task file.
For example:
.. code-block:: bash
from oslo_log import log as logging
from taskflow import task
LOG = logging.getLogger(__name__)
class Noop(task.Task):
def __init__(self, novaclient):
self.novaclient = novaclient
super(Noop, self).__init__()
def execute(self, **kwargs):
LOG.info("Custom task executed successfully..!!")
return
#. Configure custom task in Third Party Library's setup.cfg as below:
For example, Third Party Library's setup.cfg will have following entry points
.. code-block:: bash
masakari.task_flow.tasks =
custom_pre_task = <custom_task_class_path_from_third_party_library>
custom_main_task = <custom_task_class_path_from_third_party_library>
custom_post_task = <custom_task_class_path_from_third_party_library>
Note: Entry point in Third Party Library's setup.cfg should have same key as
in Masakari setup.cfg for respective failure recovery.
#. Configure custom task in Masakari's new conf file custom-recovery-methods.conf
with same name which was given in the setup.cfg to locate class path.
For example(custom task added in host auto failure config option):
.. code-block:: bash
host_auto_failure_recovery_tasks = {
'pre': ['disable_compute_service_task', 'custom_pre_task'],
'main': ['custom_main_task', 'prepare_HA_enabled_instances_task'],
'post': ['evacuate_instances_task', 'custom_post_task']}
#. If there are any configuration parameters required for custom task,
then add them into custom-recovery-methods.conf under the same
group/section where they are registered in Third Party Library.
All config parameters related to recovery method customization
should be part of newly added conf file.
Operator will be responsible to generate masakari.conf and related
configuration files by themselves.
#. Operator should ensure output of each task should be made available to
the next tasks needing them.

22
doc/source/recovery_workflow_sample_config.rst Normal file
View File

@ -0,0 +1,22 @@
===========================================================
Masakari Customized Recovery Workflow Configuration Options
===========================================================
The following is a sample Masakari recovery workflow configuration for
adaptation and use.
.. literalinclude:: _static/masakari-custom-recovery-methods.conf.sample
Minimal Configuration
=====================
#. To generate the sample custom-recovery-methods.conf file, run the following
command from the top level of the masakari directory::
$ tox -egenconfig
#. Copy sample file ``etc/masakari/masakari-custom-recovery-methods.conf.sample`` to
``/etc/masakari`` directory
#. Remove '.sample' from files ``masakari-custom-recovery-methods.conf.sample`` which
exist at ``etc/masakari``.

4
etc/masakari/masakari-customized-recovery-flow-config-generator.conf Normal file
View File

@ -0,0 +1,4 @@
[DEFAULT]
wrap_width = 80
output_file = etc/masakari/masakari-custom-recovery-methods.conf.sample
namespace = customized_recovery_flow_opts

124
masakari/conf/engine_driver.py
View File

@ -14,6 +14,7 @@
# under the License.
from oslo_config import cfg
from oslo_config import types
instance_recovery_group = cfg.OptGroup(
@ -26,6 +27,12 @@ host_recovery_group = cfg.OptGroup(
title='Host failure recovery options',
help="Configuration options for host failure recovery")
customized_recovery_flow_group = cfg.OptGroup(
'taskflow_driver_recovery_flows',
title='Customized recovery flow Options',
help="Configuration options for customizing various failure recovery"
"workflow tasks.")
host_failure_opts = [
cfg.BoolOpt('evacuate_all_instances',
@ -70,16 +77,131 @@ When set to False, it will only execute instance failure recovery actions
for an instance which contain metadata key 'HA_Enabled=True'."""),
]
taskflow_driver_recovery_flows = [
cfg.Opt('host_auto_failure_recovery_tasks',
type=types.Dict(
bounds=False,
value_type=types.List(bounds=True,
item_type=types.String(quotes=True))),
default={'pre': ['disable_compute_service_task'],
'main': ['prepare_HA_enabled_instances_task'],
'post': ['evacuate_instances_task']},
help=("""
This option allows operator to customize tasks to be executed for host failure
auto recovery workflow.
Provide list of strings reflecting to the task classes that should be included
to the host failure recovery workflow. The full classname path of all task
classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
these classes may be implemented by OpenStack Masaskari project team, deployer
or third party.
By default below three tasks will be part of this config option:-
1. disable_compute_service_task
2. prepare_HA_enabled_instances_task
3. evacuate_instances_task
The allowed values for this option is comma separated dictionary of object
names in between ``{`` and ``}``.""")),
cfg.Opt('host_rh_failure_recovery_tasks',
type=types.Dict(
bounds=False,
value_type=types.List(bounds=True,
item_type=types.String(quotes=True))),
default={'pre': ['disable_compute_service_task'],
'main': ['prepare_HA_enabled_instances_task',
'evacuate_instances_task'],
'post': []},
help=("""
This option allows operator to customize tasks to be executed for host failure
reserved_host recovery workflow.
Provide list of strings reflecting to the task classes that should be included
to the host failure recovery workflow. The full classname path of all task
classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
these classes may be implemented by OpenStack Masaskari project team, deployer
or third party.
By default below three tasks will be part of this config option:-
1. disable_compute_service_task
2. prepare_HA_enabled_instances_task
3. evacuate_instances_task
The allowed values for this option is comma separated dictionary of object
names in between ``{`` and ``}``.""")),
cfg.Opt('instance_failure_recovery_tasks',
type=types.Dict(
bounds=False,
value_type=types.List(bounds=True,
item_type=types.String(quotes=True))),
default={'pre': ['stop_instance_task'],
'main': ['start_instance_task'],
'post': ['confirm_instance_active_task']},
help=("""
This option allows operator to customize tasks to be executed for instance
failure recovery workflow.
Provide list of strings reflecting to the task classes that should be included
to the instance failure recovery workflow. The full classname path of all task
classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
these classes may be implemented by OpenStack Masaskari project team, deployer
or third party.
By default below three tasks will be part of this config option:-
1. stop_instance_task
2. start_instance_task
3. confirm_instance_active_task
The allowed values for this option is comma separated dictionary of object
names in between ``{`` and ``}``.""")),
cfg.Opt('process_failure_recovery_tasks',
type=types.Dict(
bounds=False,
value_type=types.List(bounds=True,
item_type=types.String(quotes=True))),
default={'pre': ['disable_compute_node_task'],
'main': ['confirm_compute_node_disabled_task'],
'post': []},
help=("""
This option allows operator to customize tasks to be executed for process
failure recovery workflow.
Provide list of strings reflecting to the task classes that should be included
to the process failure recovery workflow. The full classname path of all task
classes should be defined in the 'masakari.task_flow.tasks' of setup.cfg and
these classes may be implemented by OpenStack Masaskari project team, deployer
or third party.
By default below two tasks will be part of this config option:-
1. disable_compute_node_task
2. confirm_compute_node_disabled_task
The allowed values for this option is comma separated dictionary of object
names in between ``{`` and ``}``."""))
]
def register_opts(conf):
conf.register_group(instance_recovery_group)
conf.register_group(host_recovery_group)
conf.register_group(customized_recovery_flow_group)
conf.register_opts(instance_failure_options, group=instance_recovery_group)
conf.register_opts(host_failure_opts, group=host_recovery_group)
conf.register_opts(taskflow_driver_recovery_flows,
group=customized_recovery_flow_group)
def list_opts():
return {
instance_recovery_group.name: instance_failure_options,
host_recovery_group.name: host_failure_opts
host_recovery_group.name: host_failure_opts,
}
def customized_recovery_flow_list_opts():
return {
customized_recovery_flow_group.name: taskflow_driver_recovery_flows
}

12
masakari/conf/opts.py
View File

@ -31,8 +31,15 @@ import importlib
import os
import pkgutil
from masakari.conf import engine_driver
LIST_OPTS_FUNC_NAME = "list_opts"
_recovery_workflow_opts = [
('taskflow_driver_recovery_flows',
engine_driver.taskflow_driver_recovery_flows)
]
def _tupleize(dct):
"""Take the dict of options and convert to the 2-tuple format."""
@ -47,6 +54,11 @@ def list_opts():
return _tupleize(opts)
def list_recovery_workflow_opts():
"""Return a list of oslo_config options available for recovery workflow"""
return [(key, val) for key, val in _recovery_workflow_opts]
def _list_module_names():
module_names = []
package_path = os.path.dirname(os.path.abspath(__file__))

14
masakari/engine/drivers/taskflow/base.py
View File

@ -15,6 +15,7 @@
import os
from oslo_log import log as logging
from stevedore import named
# For more information please visit: https://wiki.openstack.org/wiki/TaskFlow
from taskflow import formatters
from taskflow.listeners import base
@ -88,3 +89,16 @@ class DynamicLogListener(logging_listener.DynamicLoggingListener):
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for,
log=logger, fail_formatter=SpecialFormatter(engine))
def get_recovery_flow(task_list, **kwargs):
"""This is used create extension object from provided task_list.
This method returns the extension object of the each task provided
in a list using stevedore extension manager.
"""
extensions = named.NamedExtensionManager(
'masakari.task_flow.tasks', names=task_list,
name_order=True, invoke_on_load=True, invoke_kwds=kwargs)
for extension in extensions.extensions:
yield extension.obj

53
masakari/engine/drivers/taskflow/host_failure.py
View File

@ -17,6 +17,7 @@ import eventlet
from eventlet import greenpool
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
@ -41,6 +42,8 @@ ACTION = 'instance:evacuate'
# Instance power_state
SHUTDOWN = 4
TASKFLOW_CONF = cfg.CONF.taskflow_driver_recovery_flows
class DisableComputeServiceTask(base.MasakariTask):
def __init__(self, novaclient):
@ -322,13 +325,30 @@ def get_auto_flow(novaclient, process_what):
"""
flow_name = ACTION.replace(":", "_") + "_engine"
auto_evacuate_flow = linear_flow.Flow(flow_name)
nested_flow = linear_flow.Flow(flow_name)
auto_evacuate_flow.add(DisableComputeServiceTask(novaclient),
PrepareHAEnabledInstancesTask(novaclient),
EvacuateInstancesTask(novaclient))
task_dict = TASKFLOW_CONF.host_auto_failure_recovery_tasks
return taskflow.engines.load(auto_evacuate_flow, store=process_what)
auto_evacuate_flow_pre = linear_flow.Flow('pre_tasks')
for plugin in base.get_recovery_flow(task_dict['pre'],
novaclient=novaclient):
auto_evacuate_flow_pre.add(plugin)
auto_evacuate_flow_main = linear_flow.Flow('main_tasks')
for plugin in base.get_recovery_flow(task_dict['main'],
novaclient=novaclient):
auto_evacuate_flow_main.add(plugin)
auto_evacuate_flow_post = linear_flow.Flow('post_tasks')
for plugin in base.get_recovery_flow(task_dict['post'],
novaclient=novaclient):
auto_evacuate_flow_post.add(plugin)
nested_flow.add(auto_evacuate_flow_pre)
nested_flow.add(auto_evacuate_flow_main)
nested_flow.add(auto_evacuate_flow_post)
return taskflow.engines.load(nested_flow, store=process_what)
def get_rh_flow(novaclient, process_what):
@ -344,13 +364,28 @@ def get_rh_flow(novaclient, process_what):
flow_name = ACTION.replace(":", "_") + "_engine"
nested_flow = linear_flow.Flow(flow_name)
rh_flow = linear_flow.Flow(
task_dict = TASKFLOW_CONF.host_rh_failure_recovery_tasks
rh_evacuate_flow_pre = linear_flow.Flow('pre_tasks')
for plugin in base.get_recovery_flow(task_dict['pre'],
novaclient=novaclient):
rh_evacuate_flow_pre.add(plugin)
rh_evacuate_flow_main = linear_flow.Flow(
"retry_%s" % flow_name, retry=retry.ParameterizedForEach(
rebind=['reserved_host_list'], provides='reserved_host'))
rh_flow.add(PrepareHAEnabledInstancesTask(novaclient),
EvacuateInstancesTask(novaclient))
for plugin in base.get_recovery_flow(task_dict['main'],
novaclient=novaclient):
rh_evacuate_flow_main.add(plugin)
nested_flow.add(DisableComputeServiceTask(novaclient), rh_flow)
rh_evacuate_flow_post = linear_flow.Flow('post_tasks')
for plugin in base.get_recovery_flow(task_dict['post'],
novaclient=novaclient):
rh_evacuate_flow_post.add(plugin)
nested_flow.add(rh_evacuate_flow_pre)
nested_flow.add(rh_evacuate_flow_main)
nested_flow.add(rh_evacuate_flow_post)
return taskflow.engines.load(nested_flow, store=process_what)

31
masakari/engine/drivers/taskflow/instance_failure.py
View File

@ -15,6 +15,7 @@
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import strutils
@ -33,6 +34,8 @@ LOG = logging.getLogger(__name__)
ACTION = "instance:recovery"
TASKFLOW_CONF = cfg.CONF.taskflow_driver_recovery_flows
class StopInstanceTask(base.MasakariTask):
def __init__(self, novaclient):
@ -158,11 +161,27 @@ def get_instance_recovery_flow(novaclient, process_what):
"""
flow_name = ACTION.replace(":", "_") + "_engine"
instance_recovery_workflow = linear_flow.Flow(flow_name)
nested_flow = linear_flow.Flow(flow_name)
instance_recovery_workflow.add(StopInstanceTask(novaclient),
StartInstanceTask(novaclient),
ConfirmInstanceActiveTask(novaclient))
task_dict = TASKFLOW_CONF.instance_failure_recovery_tasks
return taskflow.engines.load(instance_recovery_workflow,
store=process_what)
instance_recovery_workflow_pre = linear_flow.Flow('pre_tasks')
for plugin in base.get_recovery_flow(task_dict['pre'],
novaclient=novaclient):
instance_recovery_workflow_pre.add(plugin)
instance_recovery_workflow_main = linear_flow.Flow('main_tasks')
for plugin in base.get_recovery_flow(task_dict['main'],
novaclient=novaclient):
instance_recovery_workflow_main.add(plugin)
instance_recovery_workflow_post = linear_flow.Flow('post_tasks')
for plugin in base.get_recovery_flow(task_dict['post'],
novaclient=novaclient):
instance_recovery_workflow_post.add(plugin)
nested_flow.add(instance_recovery_workflow_pre)
nested_flow.add(instance_recovery_workflow_main)
nested_flow.add(instance_recovery_workflow_post)
return taskflow.engines.load(nested_flow, store=process_what)

30
masakari/engine/drivers/taskflow/no_op.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2018 NTT DATA.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from taskflow import task
LOG = logging.getLogger(__name__)
class Noop(task.Task):
def __init__(self, novaclient):
self.novaclient = novaclient
super(Noop, self).__init__()
def execute(self, **kwargs):
LOG.info("Custom task executed successfully..!!")
return

31
masakari/engine/drivers/taskflow/process_failure.py
View File

@ -15,6 +15,7 @@
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
import taskflow.engines
@ -32,6 +33,8 @@ LOG = logging.getLogger(__name__)
ACTION = "process:recovery"
TASKFLOW_CONF = cfg.CONF.taskflow_driver_recovery_flows
class DisableComputeNodeTask(base.MasakariTask):
def __init__(self, novaclient):
@ -93,11 +96,27 @@ def get_compute_process_recovery_flow(novaclient, process_what):
"""
flow_name = ACTION.replace(":", "_") + "_engine"
compute_process_recovery_workflow = linear_flow.Flow(flow_name)
nested_flow = linear_flow.Flow(flow_name)
compute_process_recovery_workflow.add(
DisableComputeNodeTask(novaclient),
ConfirmComputeNodeDisabledTask(novaclient))
task_dict = TASKFLOW_CONF.process_failure_recovery_tasks
return taskflow.engines.load(compute_process_recovery_workflow,
store=process_what)
process_recovery_workflow_pre = linear_flow.Flow('pre_tasks')
for plugin in base.get_recovery_flow(task_dict['pre'],
novaclient=novaclient):
process_recovery_workflow_pre.add(plugin)
process_recovery_workflow_main = linear_flow.Flow('main_tasks')
for plugin in base.get_recovery_flow(task_dict['main'],
novaclient=novaclient):
process_recovery_workflow_main.add(plugin)
process_recovery_workflow_post = linear_flow.Flow('post_tasks')
for plugin in base.get_recovery_flow(task_dict['post'],
novaclient=novaclient):
process_recovery_workflow_post.add(plugin)
nested_flow.add(process_recovery_workflow_pre)
nested_flow.add(process_recovery_workflow_main)
nested_flow.add(process_recovery_workflow_post)
return taskflow.engines.load(nested_flow, store=process_what)

111
masakari/tests/unit/engine/test_engine_mgr.py
View File

@ -20,6 +20,7 @@ from oslo_utils import timeutils
import masakari.conf
from masakari import context
from masakari import exception
from masakari.objects import fields
from masakari.objects import host as host_obj
from masakari.objects import notification as notification_obj
from masakari import test
@ -475,3 +476,113 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
self.assertEqual(expected_log, args[0])
self.assertEqual(expected_log_args_1, args[1])
@mock.patch('masakari.compute.nova.novaclient')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'DisableComputeServiceTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'PrepareHAEnabledInstancesTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'EvacuateInstancesTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.no_op.LOG')
def test_host_failure_custom_flow_for_auto_recovery(
self, _mock_log, _mock_task1, _mock_task2, _mock_task3,
_mock_novaclient, _mock_notification_get):
self.override_config(
"host_auto_failure_recovery_tasks",
{'pre': ['disable_compute_service_task', 'no_op'],
'main': ['prepare_HA_enabled_instances_task'],
'post': ['evacuate_instances_task']},
"taskflow_driver_recovery_flows")
expected_msg_format = "Custom task executed successfully..!!"
self.engine.driver.execute_host_failure(
self.context, 'fake_host',
fields.FailoverSegmentRecoveryMethod.AUTO,
uuidsentinel.fake_notification)
# Ensure custom_task added to the 'host_auto_failure_recovery_tasks'
# is executed.
_mock_log.info.assert_called_with(expected_msg_format)
@mock.patch('masakari.compute.nova.novaclient')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'DisableComputeServiceTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'PrepareHAEnabledInstancesTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.host_failure.'
'EvacuateInstancesTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.no_op.LOG')
def test_host_failure_custom_flow_for_rh_recovery(
self, _mock_log, _mock_task1, _mock_task2, _mock_task3,
_mock_novaclient, _mock_notification_get):
self.override_config(
"host_rh_failure_recovery_tasks",
{'pre': ['disable_compute_service_task'],
'main': [],
'post': ['no_op']},
"taskflow_driver_recovery_flows")
expected_msg_format = "Custom task executed successfully..!!"
self.engine.driver.execute_host_failure(
self.context, 'fake_host',
fields.FailoverSegmentRecoveryMethod.RESERVED_HOST,
uuidsentinel.fake_notification,
reserved_host_list=['host-1', 'host-2'])
# Ensure custom_task added to the 'host_rh_failure_recovery_tasks'
# is executed.
_mock_log.info.assert_called_with(expected_msg_format)
@mock.patch('masakari.compute.nova.novaclient')
@mock.patch('masakari.engine.drivers.taskflow.instance_failure.'
'StopInstanceTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.instance_failure.'
'StartInstanceTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.instance_failure.'
'ConfirmInstanceActiveTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.no_op.LOG')
def test_instance_failure_custom_flow_recovery(
self, _mock_log, _mock_task1, _mock_task2, _mock_task3,
_mock_novaclient, _mock_notification_get):
self.override_config(
"instance_failure_recovery_tasks",
{'pre': ['stop_instance_task', 'no_op'],
'main': ['start_instance_task'],
'post': ['confirm_instance_active_task']},
"taskflow_driver_recovery_flows")
expected_msg_format = "Custom task executed successfully..!!"
self.engine.driver.execute_instance_failure(
self.context, uuidsentinel.fake_ins,
uuidsentinel.fake_notification)
# Ensure custom_task added to the 'instance_failure_recovery_tasks'
# is executed.
_mock_log.info.assert_called_with(expected_msg_format)
@mock.patch('masakari.compute.nova.novaclient')
@mock.patch('masakari.engine.drivers.taskflow.process_failure.'
'DisableComputeNodeTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.process_failure.'
'ConfirmComputeNodeDisabledTask.execute')
@mock.patch('masakari.engine.drivers.taskflow.no_op.LOG')
def test_process_failure_custom_flow_recovery(
self, _mock_log, _mock_task1, _mock_task2, _mock_novaclient,
_mock_notification_get):
self.override_config(
"process_failure_recovery_tasks",
{'pre': ['disable_compute_node_task', 'no_op'],
'main': ['confirm_compute_node_disabled_task'],
'post': []},
"taskflow_driver_recovery_flows")
expected_msg_format = "Custom task executed successfully..!!"
self.engine.driver.execute_process_failure(
self.context, 'nova-compute', 'fake_host',
uuidsentinel.fake_notification)
_mock_log.info.assert_any_call(expected_msg_format)
# Ensure custom_task added to the 'process_failure_recovery_tasks'
# is executed.
_mock_log.info.assert_called_with(expected_msg_format)

20
releasenotes/notes/recovery-method-customization-3438b0e26e322b88.yaml Normal file
View File

@ -0,0 +1,20 @@
---
features:
- |
Operator can now customize workflows to process each type of failure
notifications (hosts, instance and process) as per their requirements.
Added below new config section for customized recovery flow in a new conf
file masakari-custom-recovery-methods.conf
- [taskflow_driver_recovery_flows]
Under [taskflow_driver_recovery_flows] is added below five new config options
- 'instance_failure_recovery_tasks' is a dict of tasks which will recover
instance failure.
- 'process_failure_recovery_tasks' is a dict of tasks which will recover
process failure.
- 'host_auto_failure_recovery_tasks' is a dict of tasks which will recover
host failure for auto recovery.
- 'host_rh_failure_recovery_tasks' is a dict of tasks which will recover
host failure for rh recovery on failure host.

13
setup.cfg
View File

@ -22,12 +22,14 @@ classifier =
data_files =
etc/masakari =
etc/masakari/api-paste.ini
etc/masakari/masakari-custom-recovery-methods.conf
packages =
masakari
[entry_points]
oslo.config.opts =
masakari.conf = masakari.conf.opts:list_opts
customized_recovery_flow_opts = masakari.conf.opts:list_recovery_workflow_opts
oslo.config.opts.defaults =
masakari.api = masakari.common.config:set_middleware_defaults
@ -63,6 +65,17 @@ masakari.api.v1.extensions =
masakari.driver =
taskflow_driver = masakari.engine.drivers.taskflow:TaskFlowDriver
masakari.task_flow.tasks =
disable_compute_service_task = masakari.engine.drivers.taskflow.host_failure:DisableComputeServiceTask
prepare_HA_enabled_instances_task = masakari.engine.drivers.taskflow.host_failure:PrepareHAEnabledInstancesTask
evacuate_instances_task = masakari.engine.drivers.taskflow.host_failure:EvacuateInstancesTask
stop_instance_task = masakari.engine.drivers.taskflow.instance_failure:StopInstanceTask
start_instance_task = masakari.engine.drivers.taskflow.instance_failure:StartInstanceTask
confirm_instance_active_task = masakari.engine.drivers.taskflow.instance_failure:ConfirmInstanceActiveTask
disable_compute_node_task = masakari.engine.drivers.taskflow.process_failure:DisableComputeNodeTask
confirm_compute_node_disabled_task = masakari.engine.drivers.taskflow.process_failure:ConfirmComputeNodeDisabledTask
no_op = masakari.engine.drivers.taskflow.no_op:Noop
[build_sphinx]
source-dir = doc/source
build-dir = doc/build

1
tox.ini
View File

@ -37,6 +37,7 @@ commands =
[testenv:genconfig]
basepython = python3
commands = oslo-config-generator --config-file=etc/masakari/masakari-config-generator.conf
oslo-config-generator --config-file=etc/masakari/masakari-customized-recovery-flow-config-generator.conf
[testenv:genpolicy]
commands = oslopolicy-sample-generator --config-file=etc/masakari/masakari-policy-generator.conf