Add "action_region" param for OpenStack actions

A new config item 'modules-support-region' is introduced to be used by
cloud operators, mistral will decide if add 'action_region' param to
openstack service action inputs according to that config.

Fixed an action definition for tempest tests.

TODO: Add release note.

Implements: blueprint mistral-multi-region-support

Change-Id: I0b582e9f81ab72cd05f4fae592c568f38dec6e00
This commit is contained in:
Lingxian Kong 2017-04-27 00:11:22 +12:00
parent b6de4720db
commit 8b6147d076
10 changed files with 194 additions and 56 deletions

View File

@ -67,6 +67,56 @@ class OpenStackActionGenerator(action_generator.ActionGenerator):
action_namespace = None
base_action_class = None
@classmethod
def prepare_action_inputs(cls, origin_inputs, added=[]):
"""Modify action input string.
Sometimes we need to change the default action input definition for
OpenStack actions in order to make the workflow more powerful.
Examples::
>>> prepare_action_inputs('a,b,c', added=['region=RegionOne'])
a, b, c, region=RegionOne
>>> prepare_action_inputs('a,b,c=1', added=['region=RegionOne'])
a, b, region=RegionOne, c=1
>>> prepare_action_inputs('a,b,c=1,**kwargs',
added=['region=RegionOne'])
a, b, region=RegionOne, c=1, **kwargs
>>> prepare_action_inputs('**kwargs', added=['region=RegionOne'])
region=RegionOne, **kwargs
>>> prepare_action_inputs('', added=['region=RegionOne'])
region=RegionOne
:param origin_inputs: A string consists of action inputs, separated by
comma.
:param added: (Optional) A list of params to add to input string.
:return: The new action input string.
"""
if not origin_inputs:
return ", ".join(added)
inputs = [i.strip() for i in origin_inputs.split(',')]
kwarg_index = None
for index, input in enumerate(inputs):
if "=" in input:
kwarg_index = index
if "**" in input:
kwarg_index = index - 1
kwarg_index = len(inputs) if kwarg_index is None else kwarg_index
kwarg_index = kwarg_index + 1 if kwarg_index < 0 else kwarg_index
for a in added:
if "=" not in a:
inputs.insert(0, a)
kwarg_index += 1
else:
inputs.insert(kwarg_index, a)
return ", ".join(inputs)
@classmethod
def create_action_class(cls, method_name):
if not method_name:
@ -95,6 +145,15 @@ class OpenStackActionGenerator(action_generator.ActionGenerator):
continue
arg_list = i_u.get_arg_list_as_str(client_method)
# Support specifying region for OpenStack actions.
modules = CONF.openstack_actions.modules_support_region
if cls.action_namespace in modules:
arg_list = cls.prepare_action_inputs(
arg_list,
added=['action_region=""']
)
description = i_u.get_docstring(client_method)
action_classes.append(

View File

@ -73,23 +73,26 @@ zaqarclient = _try_import('zaqarclient.queues.v2.client')
class NovaAction(base.OpenStackAction):
_service_name = 'nova'
_service_type = 'compute'
def _create_client(self):
ctx = context.ctx()
LOG.debug("Nova action security context: %s" % ctx)
keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
nova_endpoint = keystone_utils.get_endpoint_for_project('nova')
nova_endpoint = self.get_service_endpoint()
client = novaclient.Client(
2,
username=None,
api_key=None,
endpoint_type=CONF.os_actions_endpoint_type,
endpoint_type=CONF.openstack_actions.os_actions_endpoint_type,
service_type='compute',
auth_token=ctx.auth_token,
tenant_id=ctx.project_id,
region_name=keystone_endpoint.region,
region_name=nova_endpoint.region,
auth_url=keystone_endpoint.url,
insecure=ctx.insecure
)
@ -107,6 +110,7 @@ class NovaAction(base.OpenStackAction):
class GlanceAction(base.OpenStackAction):
_service_name = 'glance'
@classmethod
def _get_client_class(cls):
@ -117,7 +121,7 @@ class GlanceAction(base.OpenStackAction):
LOG.debug("Glance action security context: %s" % ctx)
glance_endpoint = keystone_utils.get_endpoint_for_project('glance')
glance_endpoint = self.get_service_endpoint()
return self._get_client_class()(
glance_endpoint.url,
@ -179,6 +183,7 @@ class KeystoneAction(base.OpenStackAction):
class CeilometerAction(base.OpenStackAction):
_service_name = 'ceilometer'
@classmethod
def _get_client_class(cls):
@ -189,9 +194,7 @@ class CeilometerAction(base.OpenStackAction):
LOG.debug("Ceilometer action security context: %s" % ctx)
ceilometer_endpoint = keystone_utils.get_endpoint_for_project(
'ceilometer'
)
ceilometer_endpoint = self.get_service_endpoint()
endpoint_url = keystone_utils.format_url(
ceilometer_endpoint.url,
@ -212,6 +215,7 @@ class CeilometerAction(base.OpenStackAction):
class HeatAction(base.OpenStackAction):
_service_name = 'heat'
@classmethod
def _get_client_class(cls):
@ -222,7 +226,7 @@ class HeatAction(base.OpenStackAction):
LOG.debug("Heat action security context: %s" % ctx)
heat_endpoint = keystone_utils.get_endpoint_for_project('heat')
heat_endpoint = self.get_service_endpoint()
endpoint_url = keystone_utils.format_url(
heat_endpoint.url,
@ -246,6 +250,7 @@ class HeatAction(base.OpenStackAction):
class NeutronAction(base.OpenStackAction):
_service_name = 'neutron'
@classmethod
def _get_client_class(cls):
@ -256,7 +261,7 @@ class NeutronAction(base.OpenStackAction):
LOG.debug("Neutron action security context: %s" % ctx)
neutron_endpoint = keystone_utils.get_endpoint_for_project('neutron')
neutron_endpoint = self.get_service_endpoint()
return self._get_client_class()(
endpoint_url=neutron_endpoint.url,
@ -268,6 +273,7 @@ class NeutronAction(base.OpenStackAction):
class CinderAction(base.OpenStackAction):
_service_type = 'volumev2'
@classmethod
def _get_client_class(cls):
@ -278,9 +284,7 @@ class CinderAction(base.OpenStackAction):
LOG.debug("Cinder action security context: %s" % ctx)
cinder_endpoint = keystone_utils.get_endpoint_for_project(
service_type='volumev2'
)
cinder_endpoint = self.get_service_endpoint()
cinder_url = keystone_utils.format_url(
cinder_endpoint.url,
@ -348,6 +352,7 @@ class MistralAction(base.OpenStackAction):
class TroveAction(base.OpenStackAction):
_service_type = 'database'
@classmethod
def _get_client_class(cls):
@ -358,9 +363,7 @@ class TroveAction(base.OpenStackAction):
LOG.debug("Trove action security context: %s" % ctx)
trove_endpoint = keystone_utils.get_endpoint_for_project(
service_type='database'
)
trove_endpoint = self.get_service_endpoint()
trove_url = keystone_utils.format_url(
trove_endpoint.url,
@ -387,6 +390,7 @@ class TroveAction(base.OpenStackAction):
class IronicAction(base.OpenStackAction):
_service_name = 'ironic'
@classmethod
def _get_client_class(cls):
@ -397,7 +401,7 @@ class IronicAction(base.OpenStackAction):
LOG.debug("Ironic action security context: %s" % ctx)
ironic_endpoint = keystone_utils.get_endpoint_for_project('ironic')
ironic_endpoint = self.get_service_endpoint()
return self._get_client_class()(
ironic_endpoint.url,
@ -679,6 +683,7 @@ class BarbicanAction(base.OpenStackAction):
class DesignateAction(base.OpenStackAction):
_service_type = 'dns'
@classmethod
def _get_client_class(cls):
@ -689,9 +694,7 @@ class DesignateAction(base.OpenStackAction):
LOG.debug("Designate action security context: %s" % ctx)
designate_endpoint = keystone_utils.get_endpoint_for_project(
service_type='dns'
)
designate_endpoint = self.get_service_endpoint()
designate_url = keystone_utils.format_url(
designate_endpoint.url,
@ -747,6 +750,7 @@ class MagnumAction(base.OpenStackAction):
class MuranoAction(base.OpenStackAction):
_service_name = 'murano'
@classmethod
def _get_client_class(cls):
@ -758,7 +762,7 @@ class MuranoAction(base.OpenStackAction):
LOG.debug("Murano action security context: %s" % ctx)
keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
murano_endpoint = keystone_utils.get_endpoint_for_project('murano')
murano_endpoint = self.get_service_endpoint()
return self._get_client_class()(
endpoint=murano_endpoint.url,
@ -775,6 +779,7 @@ class MuranoAction(base.OpenStackAction):
class TackerAction(base.OpenStackAction):
_service_name = 'tacker'
@classmethod
def _get_client_class(cls):
@ -786,7 +791,7 @@ class TackerAction(base.OpenStackAction):
LOG.debug("Tacker action security context: %s" % ctx)
keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
tacker_endpoint = keystone_utils.get_endpoint_for_project('tacker')
tacker_endpoint = self.get_service_endpoint()
return self._get_client_class()(
endpoint_url=tacker_endpoint.url,
@ -803,6 +808,7 @@ class TackerAction(base.OpenStackAction):
class SenlinAction(base.OpenStackAction):
_service_name = 'senlin'
@classmethod
def _get_client_class(cls):
@ -814,7 +820,7 @@ class SenlinAction(base.OpenStackAction):
LOG.debug("Senlin action security context: %s" % ctx)
keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
senlin_endpoint = keystone_utils.get_endpoint_for_project('senlin')
senlin_endpoint = self.get_service_endpoint()
return self._get_client_class()(
endpoint_url=senlin_endpoint.url,
@ -831,6 +837,7 @@ class SenlinAction(base.OpenStackAction):
class AodhAction(base.OpenStackAction):
_service_name = 'aodh'
@classmethod
def _get_client_class(cls):
@ -841,9 +848,7 @@ class AodhAction(base.OpenStackAction):
LOG.debug("Aodh action security context: %s" % ctx)
aodh_endpoint = keystone_utils.get_endpoint_for_project(
'aodh'
)
aodh_endpoint = self.get_service_endpoint()
endpoint_url = keystone_utils.format_url(
aodh_endpoint.url,
@ -864,6 +869,7 @@ class AodhAction(base.OpenStackAction):
class GnocchiAction(base.OpenStackAction):
_service_name = 'gnocchi'
@classmethod
def _get_client_class(cls):
@ -874,9 +880,7 @@ class GnocchiAction(base.OpenStackAction):
LOG.debug("Gnocchi action security context: %s" % ctx)
gnocchi_endpoint = keystone_utils.get_endpoint_for_project(
'gnocchi'
)
gnocchi_endpoint = self.get_service_endpoint()
endpoint_url = keystone_utils.format_url(
gnocchi_endpoint.url,

View File

@ -40,9 +40,12 @@ class OpenStackAction(base.Action):
client_method_name = None
_clients = LRUCache(100)
_lock = Lock()
_service_name = None
_service_type = None
def __init__(self, **kwargs):
self._kwargs_for_run = kwargs
self.action_region = self._kwargs_for_run.pop('action_region', None)
@abc.abstractmethod
def _create_client(self):
@ -120,6 +123,20 @@ class OpenStackAction(base.Action):
return client
def get_service_endpoint(self):
"""Get OpenStack service endpoint.
'service_name' and 'service_type' are defined in specific OpenStack
service action.
"""
endpoint = keystone_utils.get_endpoint_for_project(
service_name=self._service_name,
service_type=self._service_type,
region_name=self.action_region
)
return endpoint
def run(self):
try:
method = self._get_client_method(self._get_client())

View File

@ -106,14 +106,6 @@ rpc_response_timeout_opt = cfg.IntOpt(
help=_('Seconds to wait for a response from a call.')
)
os_endpoint_type = cfg.StrOpt(
'os-actions-endpoint-type',
default=os.environ.get('OS_ACTIONS_ENDPOINT_TYPE', 'public'),
choices=['public', 'admin', 'internal'],
help=_('Type of endpoint in identity service catalog to use for'
' communication with OpenStack services.')
)
expiration_token_duration = cfg.IntOpt(
'expiration_token_duration',
default=30,
@ -288,6 +280,24 @@ keycloak_oidc_opts = [
)
]
openstack_actions_opts = [
cfg.StrOpt(
'os-actions-endpoint-type',
default=os.environ.get('OS_ACTIONS_ENDPOINT_TYPE', 'public'),
choices=['public', 'admin', 'internal'],
deprecated_group='DEFAULT',
help=_('Type of endpoint in identity service catalog to use for'
' communication with OpenStack services.')
),
cfg.ListOpt(
'modules-support-region',
default=['nova', 'glance', 'ceilometer', 'heat', 'neutron', 'cinder',
'trove', 'ironic', 'designate', 'murano', 'tacker', 'senlin',
'aodh', 'gnocchi'],
help=_('List of module names that support region in actions.')
)
]
# note: this command line option is used only from sync_db and
# mistral-db-manage
os_actions_mapping_path = cfg.StrOpt(
@ -311,9 +321,14 @@ COORDINATION_GROUP = 'coordination'
EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'
PROFILER_GROUP = profiler.list_opts()[0][0]
KEYCLOAK_OIDC_GROUP = "keycloak_oidc"
OPENSTACK_ACTIONS_GROUP = 'openstack_actions'
CONF.register_opt(wf_trace_log_name_opt)
CONF.register_opt(auth_type_opt)
CONF.register_opt(js_impl_opt)
CONF.register_opt(rpc_impl_opt)
CONF.register_opt(rpc_response_timeout_opt)
CONF.register_opt(expiration_token_duration)
CONF.register_opts(api_opts, group=API_GROUP)
CONF.register_opts(engine_opts, group=ENGINE_GROUP)
@ -326,12 +341,8 @@ CONF.register_opts(event_engine_opts, group=EVENT_ENGINE_GROUP)
CONF.register_opts(pecan_opts, group=PECAN_GROUP)
CONF.register_opts(coordination_opts, group=COORDINATION_GROUP)
CONF.register_opts(profiler_opts, group=PROFILER_GROUP)
CONF.register_opt(js_impl_opt)
CONF.register_opt(rpc_impl_opt)
CONF.register_opt(rpc_response_timeout_opt)
CONF.register_opts(keycloak_oidc_opts, group=KEYCLOAK_OIDC_GROUP)
CONF.register_opt(os_endpoint_type)
CONF.register_opt(expiration_token_duration)
CONF.register_opts(openstack_actions_opts, group=OPENSTACK_ACTIONS_GROUP)
CLI_OPTS = [
use_debugger_opt,
@ -341,7 +352,7 @@ CLI_OPTS = [
default_group_opts = itertools.chain(
CLI_OPTS,
[wf_trace_log_name_opt, auth_type_opt, js_impl_opt, rpc_impl_opt,
os_endpoint_type, rpc_response_timeout_opt, expiration_token_duration]
rpc_response_timeout_opt, expiration_token_duration]
)
CONF.register_cli_opts(CLI_OPTS)
@ -368,6 +379,7 @@ def list_opts():
(EXECUTION_EXPIRATION_POLICY_GROUP, execution_expiration_policy_opts),
(PROFILER_GROUP, profiler_opts),
(KEYCLOAK_OIDC_GROUP, keycloak_oidc_opts),
(OPENSTACK_ACTIONS_GROUP, openstack_actions_opts),
(None, default_group_opts)
]

View File

@ -280,7 +280,7 @@ class PythonAction(Action):
if self.action_def.action_class:
self._inject_action_ctx_for_validating(input_dict)
# NOTE(xylan): Don't validate action input if action initialization
# NOTE(kong): Don't validate action input if action initialization
# method contains ** argument.
if '**' in self.action_def.input:
return

View File

@ -14,8 +14,8 @@ workflows:
nova:
type: direct
tasks:
networks_list:
action: nova.networks_list
flavors_list:
action: nova.flavors_list
publish:
result: <% task().result %>

View File

@ -18,6 +18,7 @@ from oslo_config import cfg
import mock
from mistral.actions import generator_factory
from mistral.actions.openstack.action_generator import base as generator_base
from mistral.actions.openstack import actions
from mistral import config
@ -100,6 +101,10 @@ class GeneratorTest(base.BaseTest):
self.assertTrue(issubclass(action['class'], action_cls))
self.assertEqual(method_name, action['class'].client_method_name)
modules = CONF.openstack_actions.modules_support_region
if generator_cls.action_namespace in modules:
self.assertIn('action_region', action['arg_list'])
def test_missing_module_from_mapping(self):
with _patch_openstack_action_mapping_path(RELATIVE_TEST_MAPPING_PATH):
for generator_cls in generator_factory.all_generators():
@ -129,6 +134,42 @@ class GeneratorTest(base.BaseTest):
elif cls not in (actions.GlanceAction, actions.KeystoneAction):
self.assertEqual([], action_names)
def test_prepare_action_inputs(self):
inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
'a,b,c',
added=['region=RegionOne']
)
self.assertEqual('a, b, c, region=RegionOne', inputs)
inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
'a,b,c=1',
added=['region=RegionOne']
)
self.assertEqual('a, b, region=RegionOne, c=1', inputs)
inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
'a,b,c=1,**kwargs',
added=['region=RegionOne']
)
self.assertEqual('a, b, region=RegionOne, c=1, **kwargs', inputs)
inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
'**kwargs',
added=['region=RegionOne']
)
self.assertEqual('region=RegionOne, **kwargs', inputs)
inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
'',
added=['region=RegionOne']
)
self.assertEqual('region=RegionOne', inputs)
@contextlib.contextmanager
def _patch_openstack_action_mapping_path(path):

View File

@ -15,7 +15,6 @@
import mock
from mistral.actions.openstack import actions
from mistral import config
from mistral import context as ctx
from oslotest import base
@ -47,9 +46,6 @@ class OpenStackActionTest(base.BaseTestCase):
mock_nova_endpoint,
mock_ks_endpoint_v2):
# this is the default, but be explicit
config.CONF.set_default('os_actions_endpoint_type', 'public')
test_ctx = ctx.MistralContext(
user_id=None,
project_id='1234',
@ -112,7 +108,7 @@ class OpenStackActionTest(base.BaseTestCase):
service_type='compute',
auth_token=test_ctx.auth_token,
tenant_id=test_ctx.project_id,
region_name=mock_ks_endpoint_v2().region,
region_name=mock_nova_endpoint().region,
auth_url=mock_ks_endpoint_v2().url,
insecure=test_ctx.insecure
)
@ -145,7 +141,7 @@ class OpenStackActionTest(base.BaseTestCase):
service_type='compute',
auth_token=test_ctx.auth_token,
tenant_id=test_ctx.project_id,
region_name=mock_ks_endpoint_v2().region,
region_name=mock_nova_endpoint().region,
auth_url=mock_ks_endpoint_v2().url,
insecure=test_ctx.insecure
)

View File

@ -381,7 +381,7 @@ def get_dict_from_entries(entries):
if isinstance(e, dict):
result.update(e)
else:
# NOTE(xylan): we put NotDefined here as the value of
# NOTE(kong): we put NotDefined here as the value of
# param without value specified, to distinguish from
# the valid values such as None, ''(empty string), etc.
result[e] = NotDefined

View File

@ -68,7 +68,8 @@ def client_for_trusts(trust_id):
return _admin_client(trust_id=trust_id)
def get_endpoint_for_project(service_name=None, service_type=None):
def get_endpoint_for_project(service_name=None, service_type=None,
region_name=None):
if service_name is None and service_type is None:
raise exceptions.MistralException(
"Either 'service_name' or 'service_type' must be provided."
@ -78,19 +79,27 @@ def get_endpoint_for_project(service_name=None, service_type=None):
service_catalog = obtain_service_catalog(ctx)
# When region_name is not passed, first get from context as region_name
# could be passed to rest api in http header ('X-Region-Name'). Otherwise,
# just get region from mistral configuration.
region = (region_name or ctx.region_name or
CONF.keystone_authtoken.region_name)
service_endpoints = service_catalog.get_endpoints(
service_name=service_name,
service_type=service_type,
region_name=ctx.region_name
region_name=region
)
endpoint = None
os_actions_endpoint_type = CONF.openstack_actions.os_actions_endpoint_type
for endpoints in six.itervalues(service_endpoints):
for ep in endpoints:
# is V3 interface?
if 'interface' in ep:
interface_type = ep['interface']
if CONF.os_actions_endpoint_type in interface_type:
if os_actions_endpoint_type in interface_type:
endpoint = ks_endpoints.Endpoint(
None,
ep,
@ -114,7 +123,7 @@ def get_endpoint_for_project(service_name=None, service_type=None):
raise exceptions.MistralException(
"No endpoints found [service_name=%s, service_type=%s,"
" region_name=%s]"
% (service_name, service_type, ctx.region_name)
% (service_name, service_type, region)
)
else:
return endpoint