Add "action_region" param for OpenStack actions

A new config item 'modules-support-region' is introduced to be used by
cloud operators, mistral will decide if add 'action_region' param to
openstack service action inputs according to that config.

Fixed an action definition for tempest tests.

TODO: Add release note.

Implements: blueprint mistral-multi-region-support

Change-Id: I0b582e9f81ab72cd05f4fae592c568f38dec6e00

mistral/actions/openstack/action_generator/base.py

@@ -67,6 +67,56 @@ class OpenStackActionGenerator(action_generator.ActionGenerator):
     action_namespace = None
     base_action_class = None
 
+    @classmethod
+    def prepare_action_inputs(cls, origin_inputs, added=[]):
+        """Modify action input string.
+
+        Sometimes we need to change the default action input definition for
+        OpenStack actions in order to make the workflow more powerful.
+
+        Examples::
+
+            >>> prepare_action_inputs('a,b,c', added=['region=RegionOne'])
+            a, b, c, region=RegionOne
+            >>> prepare_action_inputs('a,b,c=1', added=['region=RegionOne'])
+            a, b, region=RegionOne, c=1
+            >>> prepare_action_inputs('a,b,c=1,**kwargs',
+                                      added=['region=RegionOne'])
+            a, b, region=RegionOne, c=1, **kwargs
+            >>> prepare_action_inputs('**kwargs', added=['region=RegionOne'])
+            region=RegionOne, **kwargs
+            >>> prepare_action_inputs('', added=['region=RegionOne'])
+            region=RegionOne
+
+        :param origin_inputs: A string consists of action inputs, separated by
+            comma.
+        :param added: (Optional) A list of params to add to input string.
+        :return: The new action input string.
+        """
+        if not origin_inputs:
+            return ", ".join(added)
+
+        inputs = [i.strip() for i in origin_inputs.split(',')]
+        kwarg_index = None
+
+        for index, input in enumerate(inputs):
+            if "=" in input:
+                kwarg_index = index
+            if "**" in input:
+                kwarg_index = index - 1
+
+        kwarg_index = len(inputs) if kwarg_index is None else kwarg_index
+        kwarg_index = kwarg_index + 1 if kwarg_index < 0 else kwarg_index
+
+        for a in added:
+            if "=" not in a:
+                inputs.insert(0, a)
+                kwarg_index += 1
+            else:
+                inputs.insert(kwarg_index, a)
+
+        return ", ".join(inputs)
+
     @classmethod
     def create_action_class(cls, method_name):
         if not method_name:
@@ -95,6 +145,15 @@ class OpenStackActionGenerator(action_generator.ActionGenerator):
                 continue
 
             arg_list = i_u.get_arg_list_as_str(client_method)
+
+            # Support specifying region for OpenStack actions.
+            modules = CONF.openstack_actions.modules_support_region
+            if cls.action_namespace in modules:
+                arg_list = cls.prepare_action_inputs(
+                    arg_list,
+                    added=['action_region=""']
+                )
+
             description = i_u.get_docstring(client_method)
 
             action_classes.append(

mistral/actions/openstack/actions.py

@@ -73,23 +73,26 @@ zaqarclient = _try_import('zaqarclient.queues.v2.client')
 
 
 class NovaAction(base.OpenStackAction):
+    _service_name = 'nova'
+    _service_type = 'compute'
+
     def _create_client(self):
         ctx = context.ctx()
 
         LOG.debug("Nova action security context: %s" % ctx)
 
         keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
-        nova_endpoint = keystone_utils.get_endpoint_for_project('nova')
+        nova_endpoint = self.get_service_endpoint()
 
         client = novaclient.Client(
             2,
             username=None,
             api_key=None,
-            endpoint_type=CONF.os_actions_endpoint_type,
+            endpoint_type=CONF.openstack_actions.os_actions_endpoint_type,
             service_type='compute',
             auth_token=ctx.auth_token,
             tenant_id=ctx.project_id,
-            region_name=keystone_endpoint.region,
+            region_name=nova_endpoint.region,
             auth_url=keystone_endpoint.url,
             insecure=ctx.insecure
         )
@@ -107,6 +110,7 @@ class NovaAction(base.OpenStackAction):
 
 
 class GlanceAction(base.OpenStackAction):
+    _service_name = 'glance'
 
     @classmethod
     def _get_client_class(cls):
@@ -117,7 +121,7 @@ class GlanceAction(base.OpenStackAction):
 
         LOG.debug("Glance action security context: %s" % ctx)
 
-        glance_endpoint = keystone_utils.get_endpoint_for_project('glance')
+        glance_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             glance_endpoint.url,
@@ -179,6 +183,7 @@ class KeystoneAction(base.OpenStackAction):
 
 
 class CeilometerAction(base.OpenStackAction):
+    _service_name = 'ceilometer'
 
     @classmethod
     def _get_client_class(cls):
@@ -189,9 +194,7 @@ class CeilometerAction(base.OpenStackAction):
 
         LOG.debug("Ceilometer action security context: %s" % ctx)
 
-        ceilometer_endpoint = keystone_utils.get_endpoint_for_project(
+        ceilometer_endpoint = self.get_service_endpoint()
-            'ceilometer'
-        )
 
         endpoint_url = keystone_utils.format_url(
             ceilometer_endpoint.url,
@@ -212,6 +215,7 @@ class CeilometerAction(base.OpenStackAction):
 
 
 class HeatAction(base.OpenStackAction):
+    _service_name = 'heat'
 
     @classmethod
     def _get_client_class(cls):
@@ -222,7 +226,7 @@ class HeatAction(base.OpenStackAction):
 
         LOG.debug("Heat action security context: %s" % ctx)
 
-        heat_endpoint = keystone_utils.get_endpoint_for_project('heat')
+        heat_endpoint = self.get_service_endpoint()
 
         endpoint_url = keystone_utils.format_url(
             heat_endpoint.url,
@@ -246,6 +250,7 @@ class HeatAction(base.OpenStackAction):
 
 
 class NeutronAction(base.OpenStackAction):
+    _service_name = 'neutron'
 
     @classmethod
     def _get_client_class(cls):
@@ -256,7 +261,7 @@ class NeutronAction(base.OpenStackAction):
 
         LOG.debug("Neutron action security context: %s" % ctx)
 
-        neutron_endpoint = keystone_utils.get_endpoint_for_project('neutron')
+        neutron_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             endpoint_url=neutron_endpoint.url,
@@ -268,6 +273,7 @@ class NeutronAction(base.OpenStackAction):
 
 
 class CinderAction(base.OpenStackAction):
+    _service_type = 'volumev2'
 
     @classmethod
     def _get_client_class(cls):
@@ -278,9 +284,7 @@ class CinderAction(base.OpenStackAction):
 
         LOG.debug("Cinder action security context: %s" % ctx)
 
-        cinder_endpoint = keystone_utils.get_endpoint_for_project(
+        cinder_endpoint = self.get_service_endpoint()
-            service_type='volumev2'
-        )
 
         cinder_url = keystone_utils.format_url(
             cinder_endpoint.url,
@@ -348,6 +352,7 @@ class MistralAction(base.OpenStackAction):
 
 
 class TroveAction(base.OpenStackAction):
+    _service_type = 'database'
 
     @classmethod
     def _get_client_class(cls):
@@ -358,9 +363,7 @@ class TroveAction(base.OpenStackAction):
 
         LOG.debug("Trove action security context: %s" % ctx)
 
-        trove_endpoint = keystone_utils.get_endpoint_for_project(
+        trove_endpoint = self.get_service_endpoint()
-            service_type='database'
-        )
 
         trove_url = keystone_utils.format_url(
             trove_endpoint.url,
@@ -387,6 +390,7 @@ class TroveAction(base.OpenStackAction):
 
 
 class IronicAction(base.OpenStackAction):
+    _service_name = 'ironic'
 
     @classmethod
     def _get_client_class(cls):
@@ -397,7 +401,7 @@ class IronicAction(base.OpenStackAction):
 
         LOG.debug("Ironic action security context: %s" % ctx)
 
-        ironic_endpoint = keystone_utils.get_endpoint_for_project('ironic')
+        ironic_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             ironic_endpoint.url,
@@ -679,6 +683,7 @@ class BarbicanAction(base.OpenStackAction):
 
 
 class DesignateAction(base.OpenStackAction):
+    _service_type = 'dns'
 
     @classmethod
     def _get_client_class(cls):
@@ -689,9 +694,7 @@ class DesignateAction(base.OpenStackAction):
 
         LOG.debug("Designate action security context: %s" % ctx)
 
-        designate_endpoint = keystone_utils.get_endpoint_for_project(
+        designate_endpoint = self.get_service_endpoint()
-            service_type='dns'
-        )
 
         designate_url = keystone_utils.format_url(
             designate_endpoint.url,
@@ -747,6 +750,7 @@ class MagnumAction(base.OpenStackAction):
 
 
 class MuranoAction(base.OpenStackAction):
+    _service_name = 'murano'
 
     @classmethod
     def _get_client_class(cls):
@@ -758,7 +762,7 @@ class MuranoAction(base.OpenStackAction):
         LOG.debug("Murano action security context: %s" % ctx)
 
         keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
-        murano_endpoint = keystone_utils.get_endpoint_for_project('murano')
+        murano_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             endpoint=murano_endpoint.url,
@@ -775,6 +779,7 @@ class MuranoAction(base.OpenStackAction):
 
 
 class TackerAction(base.OpenStackAction):
+    _service_name = 'tacker'
 
     @classmethod
     def _get_client_class(cls):
@@ -786,7 +791,7 @@ class TackerAction(base.OpenStackAction):
         LOG.debug("Tacker action security context: %s" % ctx)
 
         keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
-        tacker_endpoint = keystone_utils.get_endpoint_for_project('tacker')
+        tacker_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             endpoint_url=tacker_endpoint.url,
@@ -803,6 +808,7 @@ class TackerAction(base.OpenStackAction):
 
 
 class SenlinAction(base.OpenStackAction):
+    _service_name = 'senlin'
 
     @classmethod
     def _get_client_class(cls):
@@ -814,7 +820,7 @@ class SenlinAction(base.OpenStackAction):
         LOG.debug("Senlin action security context: %s" % ctx)
 
         keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
-        senlin_endpoint = keystone_utils.get_endpoint_for_project('senlin')
+        senlin_endpoint = self.get_service_endpoint()
 
         return self._get_client_class()(
             endpoint_url=senlin_endpoint.url,
@@ -831,6 +837,7 @@ class SenlinAction(base.OpenStackAction):
 
 
 class AodhAction(base.OpenStackAction):
+    _service_name = 'aodh'
 
     @classmethod
     def _get_client_class(cls):
@@ -841,9 +848,7 @@ class AodhAction(base.OpenStackAction):
 
         LOG.debug("Aodh action security context: %s" % ctx)
 
-        aodh_endpoint = keystone_utils.get_endpoint_for_project(
+        aodh_endpoint = self.get_service_endpoint()
-            'aodh'
-        )
 
         endpoint_url = keystone_utils.format_url(
             aodh_endpoint.url,
@@ -864,6 +869,7 @@ class AodhAction(base.OpenStackAction):
 
 
 class GnocchiAction(base.OpenStackAction):
+    _service_name = 'gnocchi'
 
     @classmethod
     def _get_client_class(cls):
@@ -874,9 +880,7 @@ class GnocchiAction(base.OpenStackAction):
 
         LOG.debug("Gnocchi action security context: %s" % ctx)
 
-        gnocchi_endpoint = keystone_utils.get_endpoint_for_project(
+        gnocchi_endpoint = self.get_service_endpoint()
-            'gnocchi'
-        )
 
         endpoint_url = keystone_utils.format_url(
             gnocchi_endpoint.url,

mistral/actions/openstack/base.py

@@ -40,9 +40,12 @@ class OpenStackAction(base.Action):
     client_method_name = None
     _clients = LRUCache(100)
     _lock = Lock()
+    _service_name = None
+    _service_type = None
 
     def __init__(self, **kwargs):
         self._kwargs_for_run = kwargs
+        self.action_region = self._kwargs_for_run.pop('action_region', None)
 
     @abc.abstractmethod
     def _create_client(self):
@@ -120,6 +123,20 @@ class OpenStackAction(base.Action):
 
         return client
 
+    def get_service_endpoint(self):
+        """Get OpenStack service endpoint.
+
+        'service_name' and 'service_type' are defined in specific OpenStack
+        service action.
+        """
+        endpoint = keystone_utils.get_endpoint_for_project(
+            service_name=self._service_name,
+            service_type=self._service_type,
+            region_name=self.action_region
+        )
+
+        return endpoint
+
     def run(self):
         try:
             method = self._get_client_method(self._get_client())

mistral/config.py

@@ -106,14 +106,6 @@ rpc_response_timeout_opt = cfg.IntOpt(
     help=_('Seconds to wait for a response from a call.')
 )
 
-os_endpoint_type = cfg.StrOpt(
-    'os-actions-endpoint-type',
-    default=os.environ.get('OS_ACTIONS_ENDPOINT_TYPE', 'public'),
-    choices=['public', 'admin', 'internal'],
-    help=_('Type of endpoint in identity service catalog to use for'
-           ' communication with OpenStack services.')
-)
-
 expiration_token_duration = cfg.IntOpt(
     'expiration_token_duration',
     default=30,
@@ -288,6 +280,24 @@ keycloak_oidc_opts = [
     )
 ]
 
+openstack_actions_opts = [
+    cfg.StrOpt(
+        'os-actions-endpoint-type',
+        default=os.environ.get('OS_ACTIONS_ENDPOINT_TYPE', 'public'),
+        choices=['public', 'admin', 'internal'],
+        deprecated_group='DEFAULT',
+        help=_('Type of endpoint in identity service catalog to use for'
+               ' communication with OpenStack services.')
+    ),
+    cfg.ListOpt(
+        'modules-support-region',
+        default=['nova', 'glance', 'ceilometer', 'heat', 'neutron', 'cinder',
+                 'trove', 'ironic', 'designate', 'murano', 'tacker', 'senlin',
+                 'aodh', 'gnocchi'],
+        help=_('List of module names that support region in actions.')
+    )
+]
+
 # note: this command line option is used only from sync_db and
 # mistral-db-manage
 os_actions_mapping_path = cfg.StrOpt(
@@ -311,9 +321,14 @@ COORDINATION_GROUP = 'coordination'
 EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'
 PROFILER_GROUP = profiler.list_opts()[0][0]
 KEYCLOAK_OIDC_GROUP = "keycloak_oidc"
+OPENSTACK_ACTIONS_GROUP = 'openstack_actions'
 
 CONF.register_opt(wf_trace_log_name_opt)
 CONF.register_opt(auth_type_opt)
+CONF.register_opt(js_impl_opt)
+CONF.register_opt(rpc_impl_opt)
+CONF.register_opt(rpc_response_timeout_opt)
+CONF.register_opt(expiration_token_duration)
 
 CONF.register_opts(api_opts, group=API_GROUP)
 CONF.register_opts(engine_opts, group=ENGINE_GROUP)
@@ -326,12 +341,8 @@ CONF.register_opts(event_engine_opts, group=EVENT_ENGINE_GROUP)
 CONF.register_opts(pecan_opts, group=PECAN_GROUP)
 CONF.register_opts(coordination_opts, group=COORDINATION_GROUP)
 CONF.register_opts(profiler_opts, group=PROFILER_GROUP)
-CONF.register_opt(js_impl_opt)
-CONF.register_opt(rpc_impl_opt)
-CONF.register_opt(rpc_response_timeout_opt)
 CONF.register_opts(keycloak_oidc_opts, group=KEYCLOAK_OIDC_GROUP)
-CONF.register_opt(os_endpoint_type)
+CONF.register_opts(openstack_actions_opts, group=OPENSTACK_ACTIONS_GROUP)
-CONF.register_opt(expiration_token_duration)
 
 CLI_OPTS = [
     use_debugger_opt,
@@ -341,7 +352,7 @@ CLI_OPTS = [
 default_group_opts = itertools.chain(
     CLI_OPTS,
     [wf_trace_log_name_opt, auth_type_opt, js_impl_opt, rpc_impl_opt,
-     os_endpoint_type, rpc_response_timeout_opt, expiration_token_duration]
+     rpc_response_timeout_opt, expiration_token_duration]
 )
 
 CONF.register_cli_opts(CLI_OPTS)
@@ -368,6 +379,7 @@ def list_opts():
         (EXECUTION_EXPIRATION_POLICY_GROUP, execution_expiration_policy_opts),
         (PROFILER_GROUP, profiler_opts),
         (KEYCLOAK_OIDC_GROUP, keycloak_oidc_opts),
+        (OPENSTACK_ACTIONS_GROUP, openstack_actions_opts),
         (None, default_group_opts)
     ]
 

mistral/engine/actions.py

@@ -280,7 +280,7 @@ class PythonAction(Action):
         if self.action_def.action_class:
             self._inject_action_ctx_for_validating(input_dict)
 
-        # NOTE(xylan): Don't validate action input if action initialization
+        # NOTE(kong): Don't validate action input if action initialization
         # method contains ** argument.
         if '**' in self.action_def.input:
             return

mistral/tests/resources/openstack/action_collection_wb.yaml

@@ -14,8 +14,8 @@ workflows:
   nova:
     type: direct
     tasks:
-      networks_list:
+      flavors_list:
-        action: nova.networks_list
+        action: nova.flavors_list
         publish:
            result: <% task().result %>
 

mistral/tests/unit/actions/openstack/test_generator.py

@@ -18,6 +18,7 @@ from oslo_config import cfg
 import mock
 
 from mistral.actions import generator_factory
+from mistral.actions.openstack.action_generator import base as generator_base
 from mistral.actions.openstack import actions
 from mistral import config
 
@@ -100,6 +101,10 @@ class GeneratorTest(base.BaseTest):
             self.assertTrue(issubclass(action['class'], action_cls))
             self.assertEqual(method_name, action['class'].client_method_name)
 
+            modules = CONF.openstack_actions.modules_support_region
+            if generator_cls.action_namespace in modules:
+                self.assertIn('action_region', action['arg_list'])
+
     def test_missing_module_from_mapping(self):
         with _patch_openstack_action_mapping_path(RELATIVE_TEST_MAPPING_PATH):
             for generator_cls in generator_factory.all_generators():
@@ -129,6 +134,42 @@ class GeneratorTest(base.BaseTest):
                 elif cls not in (actions.GlanceAction, actions.KeystoneAction):
                     self.assertEqual([], action_names)
 
+    def test_prepare_action_inputs(self):
+        inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
+            'a,b,c',
+            added=['region=RegionOne']
+        )
+
+        self.assertEqual('a, b, c, region=RegionOne', inputs)
+
+        inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
+            'a,b,c=1',
+            added=['region=RegionOne']
+        )
+
+        self.assertEqual('a, b, region=RegionOne, c=1', inputs)
+
+        inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
+            'a,b,c=1,**kwargs',
+            added=['region=RegionOne']
+        )
+
+        self.assertEqual('a, b, region=RegionOne, c=1, **kwargs', inputs)
+
+        inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
+            '**kwargs',
+            added=['region=RegionOne']
+        )
+
+        self.assertEqual('region=RegionOne, **kwargs', inputs)
+
+        inputs = generator_base.OpenStackActionGenerator.prepare_action_inputs(
+            '',
+            added=['region=RegionOne']
+        )
+
+        self.assertEqual('region=RegionOne', inputs)
+
 
 @contextlib.contextmanager
 def _patch_openstack_action_mapping_path(path):

mistral/tests/unit/actions/openstack/test_openstack_actions.py

@@ -15,7 +15,6 @@
 import mock
 
 from mistral.actions.openstack import actions
-from mistral import config
 from mistral import context as ctx
 from oslotest import base
 
@@ -47,9 +46,6 @@ class OpenStackActionTest(base.BaseTestCase):
                                          mock_nova_endpoint,
                                          mock_ks_endpoint_v2):
 
-        # this is the default, but be explicit
-        config.CONF.set_default('os_actions_endpoint_type', 'public')
-
         test_ctx = ctx.MistralContext(
             user_id=None,
             project_id='1234',
@@ -112,7 +108,7 @@ class OpenStackActionTest(base.BaseTestCase):
             service_type='compute',
             auth_token=test_ctx.auth_token,
             tenant_id=test_ctx.project_id,
-            region_name=mock_ks_endpoint_v2().region,
+            region_name=mock_nova_endpoint().region,
             auth_url=mock_ks_endpoint_v2().url,
             insecure=test_ctx.insecure
         )
@@ -145,7 +141,7 @@ class OpenStackActionTest(base.BaseTestCase):
             service_type='compute',
             auth_token=test_ctx.auth_token,
             tenant_id=test_ctx.project_id,
-            region_name=mock_ks_endpoint_v2().region,
+            region_name=mock_nova_endpoint().region,
             auth_url=mock_ks_endpoint_v2().url,
             insecure=test_ctx.insecure
         )

mistral/utils/__init__.py

@@ -381,7 +381,7 @@ def get_dict_from_entries(entries):
         if isinstance(e, dict):
             result.update(e)
         else:
-            # NOTE(xylan): we put NotDefined here as the value of
+            # NOTE(kong): we put NotDefined here as the value of
             # param without value specified, to distinguish from
             # the valid values such as None, ''(empty string), etc.
             result[e] = NotDefined

mistral/utils/openstack/keystone.py

@@ -68,7 +68,8 @@ def client_for_trusts(trust_id):
     return _admin_client(trust_id=trust_id)
 
 
-def get_endpoint_for_project(service_name=None, service_type=None):
+def get_endpoint_for_project(service_name=None, service_type=None,
+                             region_name=None):
     if service_name is None and service_type is None:
         raise exceptions.MistralException(
             "Either 'service_name' or 'service_type' must be provided."
@@ -78,19 +79,27 @@ def get_endpoint_for_project(service_name=None, service_type=None):
 
     service_catalog = obtain_service_catalog(ctx)
 
+    # When region_name is not passed, first get from context as region_name
+    # could be passed to rest api in http header ('X-Region-Name'). Otherwise,
+    # just get region from mistral configuration.
+    region = (region_name or ctx.region_name or
+              CONF.keystone_authtoken.region_name)
+
     service_endpoints = service_catalog.get_endpoints(
         service_name=service_name,
         service_type=service_type,
-        region_name=ctx.region_name
+        region_name=region
     )
 
     endpoint = None
+    os_actions_endpoint_type = CONF.openstack_actions.os_actions_endpoint_type
+
     for endpoints in six.itervalues(service_endpoints):
         for ep in endpoints:
             # is V3 interface?
             if 'interface' in ep:
                 interface_type = ep['interface']
-                if CONF.os_actions_endpoint_type in interface_type:
+                if os_actions_endpoint_type in interface_type:
                     endpoint = ks_endpoints.Endpoint(
                         None,
                         ep,
@@ -114,7 +123,7 @@ def get_endpoint_for_project(service_name=None, service_type=None):
         raise exceptions.MistralException(
             "No endpoints found [service_name=%s, service_type=%s,"
             " region_name=%s]"
-            % (service_name, service_type, ctx.region_name)
+            % (service_name, service_type, region)
         )
     else:
         return endpoint