Check if required services are available before starting the scenario

In this patch:
   * validation function benchmark.validation.required_services()
   * a piece of refactoring in scenarios.base and runners.base modules
   * new decorator benchmark.validation.validator(),
     which transforms validation function into scenario validator
   * constants for OpenStack services names and types
   * refactoring in rally.osclients
   * method osclients.Clients.services() to obtain available services
   * services validation is added to scenarios

blueprint check-service-availability-before-start-scenario

Change-Id: Id6126efb1bda73dd9ff92346ddf690329765f43f

rally/benchmark/context/base.py

@@ -85,12 +85,13 @@ class ContextManager(object):
     """Create context environment and run method inside it."""
 
     @staticmethod
-    def run(context, func, *args, **kwargs):
+    def run(context, func, cls, method_name, args):
         ctxlst = [Context.get_by_name(name) for name in context["config"]]
         ctxlst = map(lambda ctx: ctx(context),
                      sorted(ctxlst, key=lambda x: x.__ctx_order__))
 
-        return ContextManager._magic(ctxlst, func, *args, **kwargs)
+        return ContextManager._magic(ctxlst, func, cls,
+                                     method_name, context, args)
 
     @staticmethod
     def validate(context, non_hidden=False):
@@ -104,7 +105,7 @@ class ContextManager(object):
                                                         users=users, task=task)
 
     @staticmethod
-    def _magic(ctxlst, func, *args, **kwargs):
+    def _magic(ctxlst, func, *args):
         """Some kind of contextlib.nested but with black jack & recursion.
 
         This method uses recursion to build nested "with" from list of context
@@ -114,16 +115,15 @@ class ContextManager(object):
         :param ctxlst: list of instances of subclasses of Context
         :param func: function that will be called inside this context
         :param args: args that will be passed to function `func`
-        :param kwargs: kwargs that will be passed to function `func`
         :returns: result of function call
         """
         if not ctxlst:
-            return func(*args, **kwargs)
+            return func(*args)
 
         with ctxlst[0]:
             # TODO(boris-42): call of setup could be moved inside __enter__
             #                 but it should be in try-except, and in except
             #                 we should call by hand __exit__
             ctxlst[0].setup()
-            tmp = ContextManager._magic(ctxlst[1:], func, *args, **kwargs)
+            tmp = ContextManager._magic(ctxlst[1:], func, *args)
             return tmp

rally/benchmark/runners/base.py

@@ -209,8 +209,7 @@ class ScenarioRunner(object):
 
         args = cls.preprocess(method_name, context_obj, args)
         results = base_ctx.ContextManager.run(context_obj, self._run_scenario,
-                                              cls, method_name, context_obj,
+                                              cls, method_name, args)
-                                              args)
 
         if not isinstance(results, ScenarioRunnerResult):
             name = self.__execution_type__

rally/benchmark/scenarios/base.py

@@ -104,10 +104,10 @@ class Scenario(object):
             if not result.is_valid:
                 raise exceptions.InvalidScenarioArgument(message=result.msg)
 
-    @staticmethod
+    @classmethod
-    def validate(name, args, admin=None, users=None, task=None):
+    def validate(cls, name, args, admin=None, users=None, task=None):
         """Semantic check of benchmark arguments."""
-        validators = Scenario.meta(name, "validators", default=[])
+        validators = cls.meta(name, "validators", default=[])
 
         if not validators:
             return
@@ -120,10 +120,10 @@ class Scenario(object):
         # NOTE(boris-42): Potential bug, what if we don't have "admin" client
         #                 and scenario have "admin" validators.
         if admin:
-            Scenario._validate_helper(admin_validators, admin, args, task)
+            cls._validate_helper(admin_validators, admin, args, task)
         if users:
             for user in users:
-                Scenario._validate_helper(user_validators, user, args, task)
+                cls._validate_helper(user_validators, user, args, task)
 
     @staticmethod
     def meta(cls, attr_name, method_name=None, default=None):

rally/benchmark/scenarios/ceilometer/alarms.py

@@ -14,10 +14,13 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.ceilometer import utils as ceilometerutils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CeilometerAlarms(ceilometerutils.CeilometerScenario):
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_alarm(self, meter_name, threshold, **kwargs):
         """Test creating an alarm.
 
@@ -32,6 +35,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._create_alarm(meter_name, threshold, kwargs)
 
     @base.scenario()
+    @validation.required_services(consts.Service.CEILOMETER)
     def list_alarms(self):
         """Test fetching all alarms.
 
@@ -40,6 +44,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._list_alarms()
 
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_list_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and getting newly created alarm.
 
@@ -56,6 +61,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._list_alarms(alarm.alarm_id)
 
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_update_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and updating the newly created alarm.
 
@@ -73,6 +79,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._update_alarm(alarm.alarm_id, alarm_dict_diff)
 
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_delete_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and deleting the newly created alarm.
 

rally/benchmark/scenarios/ceilometer/meters.py

@@ -14,10 +14,13 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.ceilometer import utils as ceilometerutils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CeilometerMeters(ceilometerutils.CeilometerScenario):
     @base.scenario()
+    @validation.required_services(consts.Service.CEILOMETER)
     def list_meters(self):
         """Test fetching user's meters."""
         self._list_meters()

rally/benchmark/scenarios/ceilometer/queries.py

@@ -16,10 +16,13 @@ import json
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.ceilometer import utils as ceilometerutils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CeilometerQueries(ceilometerutils.CeilometerScenario):
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_query_alarms(self, meter_name, threshold, filter=None,
                                 orderby=None, limit=None, **kwargs):
         """Creates an alarm and then queries it with specific parameters.
@@ -40,6 +43,7 @@ class CeilometerQueries(ceilometerutils.CeilometerScenario):
         self._query_alarms(filter, orderby, limit)
 
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_query_alarm_history(self, meter_name, threshold,
                                        orderby=None, limit=None, **kwargs):
         """Creates an alarm and then queries for its history
@@ -58,6 +62,7 @@ class CeilometerQueries(ceilometerutils.CeilometerScenario):
         self._query_alarm_history(alarm_filter, orderby, limit)
 
     @base.scenario()
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_and_query_samples(self, counter_name, counter_type,
                                  counter_unit, counter_volume, resource_id,
                                  filter=None, orderby=None, limit=None,

rally/benchmark/scenarios/ceilometer/resources.py

@@ -14,10 +14,13 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.ceilometer import utils as ceilometerutils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CeilometerResource(ceilometerutils.CeilometerScenario):
     @base.scenario()
+    @validation.required_services(consts.Service.CEILOMETER)
     def list_resources(self):
         """Test fetching all resources.
 

rally/benchmark/scenarios/ceilometer/stats.py

@@ -14,10 +14,13 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.ceilometer import utils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CeilometerStats(utils.CeilometerScenario):
     @base.scenario(context={"cleanup": ["ceilometer"]})
+    @validation.required_services(consts.Service.CEILOMETER)
     def create_meter_and_get_stats(self, **kwargs):
         """Test creating a meter and fetching its statistics.
 

rally/benchmark/scenarios/cinder/volumes.py

@@ -15,11 +15,14 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.cinder import utils
+from rally.benchmark import validation
+from rally import consts
 
 
 class CinderVolumes(utils.CinderScenario):
 
     @base.scenario(context={"cleanup": ["cinder"]})
+    @validation.required_services(consts.Service.CINDER)
     def create_and_list_volume(self, size, detailed=True, **kwargs):
         """Tests creating a volume and listing volumes.
 
@@ -37,6 +40,7 @@ class CinderVolumes(utils.CinderScenario):
         self._list_volumes(detailed)
 
     @base.scenario(context={"cleanup": ["cinder"]})
+    @validation.required_services(consts.Service.CINDER)
     def create_and_delete_volume(self, size, min_sleep=0, max_sleep=0,
                                  **kwargs):
         """Tests creating and then deleting a volume.
@@ -49,6 +53,7 @@ class CinderVolumes(utils.CinderScenario):
         self._delete_volume(volume)
 
     @base.scenario(context={"cleanup": ["cinder"]})
+    @validation.required_services(consts.Service.CINDER)
     def create_volume(self, size, **kwargs):
         """Test creating volumes perfromance.
 

rally/benchmark/scenarios/glance/images.py

@@ -18,6 +18,7 @@ from rally.benchmark.scenarios.glance import utils
 from rally.benchmark.scenarios.nova import utils as nova_utils
 from rally.benchmark import types as types
 from rally.benchmark import validation
+from rally import consts
 
 
 class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
@@ -26,6 +27,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
     RESOURCE_NAME_LENGTH = 16
 
     @base.scenario(context={"cleanup": ["glance"]})
+    @validation.required_services(consts.Service.GLANCE)
     def create_and_list_image(self, container_format,
                               image_location, disk_format, **kwargs):
         """Test adding an image and then listing all images.
@@ -47,6 +49,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
         self._list_images()
 
     @base.scenario(context={"cleanup": ["glance"]})
+    @validation.required_services(consts.Service.GLANCE)
     def list_images(self):
         """Test the glance image-list command.
 
@@ -61,6 +64,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
         self._list_images()
 
     @base.scenario(context={"cleanup": ["glance"]})
+    @validation.required_services(consts.Service.GLANCE)
     def create_and_delete_image(self, container_format,
                                 image_location, disk_format, **kwargs):
         """Test adds and then deletes image."""
@@ -74,6 +78,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
 
     @types.set(flavor=types.FlavorResourceType)
     @validation.add(validation.flavor_exists("flavor"))
+    @validation.required_services(consts.Service.GLANCE, consts.Service.NOVA)
     @base.scenario(context={"cleanup": ["glance", "nova"]})
     def create_image_and_boot_instances(self, container_format,
                                         image_location, disk_format,

rally/benchmark/scenarios/heat/stacks.py

@@ -15,6 +15,8 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.heat import utils
+from rally.benchmark import validation
+from rally import consts
 
 
 class HeatStacks(utils.HeatScenario):
@@ -35,6 +37,7 @@ class HeatStacks(utils.HeatScenario):
 
     @base.scenario(context={"cleanup": ["heat"],
                             "roles": ["heat_stack_owner"]})
+    @validation.required_services(consts.Service.HEAT)
     def create_and_list_stack(self, template_path=None):
         """Test adding an stack and then listing all stacks.
 
@@ -53,6 +56,7 @@ class HeatStacks(utils.HeatScenario):
 
     @base.scenario(context={"cleanup": ["heat"],
                             "roles": ["heat_stack_owner"]})
+    @validation.required_services(consts.Service.HEAT)
     def create_and_delete_stack(self, template_path=None):
         """Test adds and then deletes stack.
 

rally/benchmark/scenarios/neutron/network.py

@@ -16,11 +16,13 @@
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.neutron import utils
 from rally.benchmark import validation
+from rally import consts
 
 
 class NeutronNetworks(utils.NeutronScenario):
 
     @base.scenario(context={"cleanup": ["neutron"]})
+    @validation.required_services(consts.Service.NEUTRON)
     def create_and_list_networks(self, network_create_args=None):
         """Create a network and then listing all networks.
 
@@ -40,6 +42,7 @@ class NeutronNetworks(utils.NeutronScenario):
 
     @base.scenario(context={"cleanup": ["neutron"]})
     @validation.add(validation.required_parameters(['subnets_per_network']))
+    @validation.required_services(consts.Service.NEUTRON)
     def create_and_list_subnets(self,
                                 network_create_args=None,
                                 subnet_create_args=None,
@@ -65,6 +68,7 @@ class NeutronNetworks(utils.NeutronScenario):
 
     @base.scenario(context={"cleanup": ["neutron"]})
     @validation.add(validation.required_parameters(['subnets_per_network']))
+    @validation.required_services(consts.Service.NEUTRON)
     def create_and_list_routers(self,
                                 network_create_args=None,
                                 subnet_create_args=None,
@@ -96,6 +100,7 @@ class NeutronNetworks(utils.NeutronScenario):
 
     @base.scenario(context={"cleanup": ["neutron"]})
     @validation.add(validation.required_parameters(["ports_per_network"]))
+    @validation.required_services(consts.Service.NEUTRON)
     def create_and_list_ports(self,
                               network_create_args=None,
                               port_create_args=None,

rally/benchmark/scenarios/nova/servers.py

@@ -23,6 +23,7 @@ from rally.benchmark.scenarios.nova import utils
 from rally.benchmark.scenarios import utils as scenario_utils
 from rally.benchmark import types as types
 from rally.benchmark import validation
+from rally import consts
 from rally import exceptions as rally_exceptions
 from rally.openstack.common.gettextutils import _  # noqa
 from rally.openstack.common import log as logging
@@ -44,6 +45,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova"]})
+    @validation.required_services(consts.Service.NOVA)
     def boot_and_list_server(self, image, flavor,
                              detailed=True, **kwargs):
         """Tests booting an image and then listing servers.
@@ -65,6 +67,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova"]})
+    @validation.required_services(consts.Service.NOVA)
     def boot_and_delete_server(self, image, flavor,
                                min_sleep=0, max_sleep=0, **kwargs):
         """Tests booting and then deleting an image."""
@@ -77,6 +80,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova", "cinder"]})
+    @validation.required_services(consts.Service.NOVA, consts.Service.CINDER)
     def boot_server_from_volume_and_delete(self, image, flavor,
                                            volume_size,
                                            min_sleep=0, max_sleep=0, **kwargs):
@@ -94,6 +98,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova"]})
+    @validation.required_services(consts.Service.NOVA)
     def boot_and_bounce_server(self, image, flavor, **kwargs):
         """Test booting a server with further performing specified actions.
 
@@ -119,6 +124,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova", "glance"]})
+    @validation.required_services(consts.Service.NOVA, consts.Service.GLANCE)
     def snapshot_server(self, image, flavor, **kwargs):
         """Tests Nova instance snapshotting."""
         server_name = self._generate_random_name()
@@ -135,6 +141,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova"]})
+    @validation.required_services(consts.Service.NOVA)
     def boot_server(self, image, flavor, **kwargs):
         """Test VM boot - assumed clean-up is done elsewhere."""
         if 'nics' not in kwargs:
@@ -149,6 +156,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.add(validation.image_valid_on_flavor("flavor", "image"))
     @base.scenario(context={"cleanup": ["nova", "cinder"]})
+    @validation.required_services(consts.Service.NOVA, consts.Service.CINDER)
     def boot_server_from_volume(self, image, flavor,
                                 volume_size, **kwargs):
         """Test VM boot from volume - assumed clean-up is done elsewhere."""

rally/benchmark/scenarios/vm/vmtasks.py

@@ -20,6 +20,7 @@ from rally.benchmark.scenarios.nova import utils as nova_utils
 from rally.benchmark.scenarios.vm import utils as vm_utils
 from rally.benchmark import types as types
 from rally.benchmark import validation
+from rally import consts
 from rally import exceptions
 
 
@@ -38,6 +39,7 @@ class VMTasks(nova_utils.NovaScenario, vm_utils.VMScenario):
                                                        "use_floatingip"))
     @base.scenario(context={"cleanup": ["nova"],
                    "keypair": {}, "allow_ssh": {}})
+    @validation.required_services(consts.Service.NOVA)
     def boot_runcommand_delete(self, image, flavor,
                                script, interpreter, username,
                                fixed_network="private",

rally/benchmark/validation.py

@@ -32,6 +32,35 @@ class ValidationResult(object):
         self.msg = msg
 
 
+def validator(fn):
+    """Decorator that constructs a scenario validator from given function.
+
+    Decorated function should return ValidationResult on error.
+
+    :param fn: function that performs validation
+    :returns: rally scenario validator
+    """
+    def wrap_given(*args, **kwargs):
+        def wrap_validator(**options):
+            options.update({"args": args, "kwargs": kwargs})
+            # NOTE(amaretskiy): validator is successful by default
+            return fn(*args, **options) or ValidationResult()
+
+        def wrap_scenario(scenario):
+            # NOTE(amaretskiy): user permission by default
+            wrap_validator.permission = getattr(
+                fn, "permission", consts.EndpointPermission.USER)
+            if not hasattr(scenario, "validators"):
+                scenario.validators = []
+            scenario.validators.append(wrap_validator)
+            return scenario
+
+        return wrap_scenario
+
+    return wrap_given
+
+
+# NOTE(amaretskiy): Deprecated by validator()
 def add(validator):
     def wrapper(func):
         if not getattr(func, 'validators', None):
@@ -324,3 +353,18 @@ def required_parameters(params):
             return ValidationResult(False, message)
         return ValidationResult()
     return required_parameters_validator
+
+
+@validator
+def required_services(*args, **kwargs):
+    """Check if specified services are available.
+
+    :param args: list of servives names
+    """
+    available_services = kwargs.get("clients").services().values()
+    for service in args:
+        if service not in consts.Service:
+            return ValidationResult(False, _("Unknown service: %s") % service)
+        if service not in available_services:
+            return ValidationResult(
+                False, _("Service is not available: %s") % service)

rally/consts.py

@@ -74,7 +74,70 @@ class _RunnerType(utils.ImmutableMixin, utils.EnumMixin):
     PERIODIC = "periodic"
 
 
+class _Service(utils.ImmutableMixin, utils.EnumMixin):
+    """OpenStack services names, by rally convention."""
+
+    NOVA = "nova"
+    NOVAV3 = "novav3"
+    CINDER = "cinder"
+    CINDERV2 = "cinderv2"
+    EC2 = "ec2"
+    GLANCE = "glance"
+    CLOUD = "cloud"
+    HEAT = "heat"
+    KEYSTONE = "keystone"
+    NEUTRON = "neutron"
+    CEILOMETER = "ceilometer"
+    S3 = "s3"
+    TROVE = "trove"
+
+
+class _ServiceType(utils.ImmutableMixin, utils.EnumMixin):
+    """OpenStack services types, mapped to service names."""
+
+    VOLUME = "volume"
+    VOLUMEV2 = "volumev2"
+    EC2 = "ec2"
+    IMAGE = "image"
+    CLOUD = "cloudformation"
+    ORCHESTRATION = "orchestration"
+    IDENTITY = "identity"
+    COMPUTE = "compute"
+    COMPUTEV3 = "computev3"
+    NETWORK = "network"
+    METERING = "metering"
+    S3 = "s3"
+    DATABASE = "database"
+
+    def __init__(self):
+        self.__names = {
+            self.COMPUTE: _Service.NOVA,
+            self.COMPUTEV3: _Service.NOVAV3,
+            self.VOLUME: _Service.CINDER,
+            self.VOLUMEV2: _Service.CINDER,
+            self.EC2: _Service.EC2,
+            self.IMAGE: _Service.GLANCE,
+            self.CLOUD: _Service.CLOUD,
+            self.ORCHESTRATION: _Service.HEAT,
+            self.IDENTITY: _Service.KEYSTONE,
+            self.NETWORK: _Service.NEUTRON,
+            self.METERING: _Service.CEILOMETER,
+            self.S3: _Service.S3,
+            self.DATABASE: _Service.TROVE
+        }
+
+    def __getitem__(self, service_type):
+        """Mapping protocol to service names.
+
+        :param name: str, service name
+        :returns: str, service type
+        """
+        return self.__names[service_type]
+
+
 TaskStatus = _TaskStatus()
 DeployStatus = _DeployStatus()
 EndpointPermission = _EndpointPermission()
 RunnerType = _RunnerType()
+ServiceType = _ServiceType()
+Service = _Service()

rally/osclients.py

@@ -26,6 +26,7 @@ from neutronclient.neutron import client as neutron
 from novaclient import client as nova
 from oslo.config import cfg
 
+from rally import consts
 from rally import exceptions
 
 
@@ -44,6 +45,20 @@ CONF.register_opts([
 nova._adapter_pool = lambda x: nova.adapters.HTTPAdapter()
 
 
+def cached(func):
+    """Cache client handles."""
+    def wrapper(self, *args, **kwargs):
+        key = '{0}{1}{2}'.format(func.__name__,
+                                 str(args) if args else '',
+                                 str(kwargs) if kwargs else '')
+
+        if key in self.cache:
+            return self.cache[key]
+        self.cache[key] = func(self, *args, **kwargs)
+        return self.cache[key]
+    return wrapper
+
+
 class Clients(object):
     """This class simplify and unify work with openstack python clients."""
 
@@ -55,21 +70,7 @@ class Clients(object):
         """Remove all cached client handles."""
         self.cache = {}
 
-    def memoize(name):
+    @cached
-        """Cache client handles."""
-        def decorate(func):
-            def wrapper(self, *args, **kwargs):
-                key = '{0}{1}{2}'.format(func.__name__,
-                                         str(args) if args else '',
-                                         str(kwargs) if kwargs else '')
-                if key in self.cache:
-                    return self.cache[key]
-                self.cache[key] = func(self, *args, **kwargs)
-                return self.cache[key]
-            return wrapper
-        return decorate
-
-    @memoize('keystone')
     def keystone(self):
         """Return keystone client."""
         new_kw = {
@@ -111,9 +112,9 @@ class Clients(object):
                 url=self.endpoint.auth_url)
         return client
 
-    @memoize('nova')
+    @cached
     def nova(self, version='2'):
-        """Returns nova client."""
+        """Return nova client."""
         client = nova.Client(version,
                              self.endpoint.username,
                              self.endpoint.password,
@@ -127,9 +128,9 @@ class Clients(object):
                              cacert=CONF.https_cacert)
         return client
 
-    @memoize('neutron')
+    @cached
     def neutron(self, version='2.0'):
-        """Returns neutron client."""
+        """Return neutron client."""
         client = neutron.Client(version,
                                 username=self.endpoint.username,
                                 password=self.endpoint.password,
@@ -141,9 +142,9 @@ class Clients(object):
                                 cacert=CONF.https_cacert)
         return client
 
-    @memoize('glance')
+    @cached
     def glance(self, version='1'):
-        """Returns glance client."""
+        """Return glance client."""
         kc = self.keystone()
         endpoint = kc.service_catalog.get_endpoints()['image'][0]
         client = glance.Client(version,
@@ -155,9 +156,9 @@ class Clients(object):
                                cacert=CONF.https_cacert)
         return client
 
-    @memoize('heat')
+    @cached
     def heat(self, version='1'):
-        """Returns heat client."""
+        """Return heat client."""
         kc = self.keystone()
         endpoint = kc.service_catalog.get_endpoints()['orchestration'][0]
 
@@ -170,9 +171,9 @@ class Clients(object):
                              cacert=CONF.https_cacert)
         return client
 
-    @memoize('cinder')
+    @cached
     def cinder(self, version='1'):
-        """Returns cinder client."""
+        """Return cinder client."""
         client = cinder.Client(version,
                                self.endpoint.username,
                                self.endpoint.password,
@@ -186,9 +187,9 @@ class Clients(object):
                                cacert=CONF.https_cacert)
         return client
 
-    @memoize('ceilometer')
+    @cached
     def ceilometer(self, version='2'):
-        """Returns ceilometer client."""
+        """Return ceilometer client."""
         kc = self.keystone()
         endpoint = kc.service_catalog.get_endpoints()['metering'][0]
         auth_token = kc.auth_token
@@ -205,9 +206,9 @@ class Clients(object):
                                    cacert=CONF.https_cacert)
         return client
 
-    @memoize('ironic')
+    @cached
     def ironic(self, version='1.0'):
-        """Returns Ironic client."""
+        """Return Ironic client."""
         client = ironic.Client(version,
                                username=self.endpoint.username,
                                password=self.endpoint.password,
@@ -217,3 +218,16 @@ class Clients(object):
                                insecure=CONF.https_insecure,
                                cacert=CONF.https_cacert)
         return client
+
+    @cached
+    def services(self):
+        """Return available services names and types.
+
+        :returns: dict, {"service_type": "service_name", ...}
+        """
+        services_data = {}
+        available_services = self.keystone().service_catalog.get_endpoints()
+        for service_type in available_services.keys():
+            if service_type in consts.ServiceType:
+                services_data[service_type] = consts.ServiceType[service_type]
+        return services_data

tests/benchmark/context/test_base.py

@@ -125,7 +125,8 @@ class ContextManagerTestCase(test.TestCase):
 
         mock_magic.return_value = 5
 
-        result = base.ContextManager.run(context, lambda x, y: x + y, 1, 2)
+        result = base.ContextManager.run(context, lambda x, y: x + y, type,
+                                         "fake_method", {"fake": "value"})
         self.assertEqual(result, 5)
 
         mock_get.assert_has_calls([

tests/benchmark/runners/test_base.py

@@ -229,8 +229,8 @@ class ScenarioRunnerTestCase(test.TestCase):
             }
         }
 
-        expected = [context_obj, runner._run_scenario, cls, method_name,
+        expected = [context_obj, runner._run_scenario, cls,
-                    context_obj, config_kwargs]
+                    method_name, config_kwargs]
         mock_ctx_manager.run.assert_called_once_with(*expected)
 
     @mock.patch("rally.benchmark.runners.base.base_ctx.ContextManager")

tests/benchmark/test_validation.py

@@ -20,6 +20,7 @@ import mock
 from novaclient import exceptions as nova_exc
 
 from rally.benchmark import validation
+from rally import consts
 from rally.openstack.common.gettextutils import _
 from tests import fakes
 from tests import test
@@ -30,6 +31,84 @@ TEMPEST = "rally.verification.verifiers.tempest.tempest"
 
 class ValidationUtilsTestCase(test.TestCase):
 
+    def _get_scenario_validators(self, func_, scenario_, reset=True):
+        """Unwrap scenario validators created by validation.validator()."""
+        if reset:
+            if hasattr(func_, "permission"):
+                del func_.permission
+            if hasattr(scenario_, "validators"):
+                del scenario_.validators
+        scenario = validation.validator(func_)()(scenario_)
+        return scenario.validators
+
+    def test_validator(self):
+
+        failure = validation.ValidationResult(False)
+        func = lambda *args, **kv: kv
+        scenario = lambda: None
+
+        # Check arguments passed to validator
+        wrap = validation.validator(func)
+        wrap_args = ["foo", "bar"]
+        wrap_kwargs = {"foo": "spam"}
+        wrap_scenario = wrap(*wrap_args, **wrap_kwargs)
+        wrap_validator = wrap_scenario(scenario)
+        validators = wrap_validator.validators
+        self.assertEqual(1, len(validators))
+        validator, = validators
+        self.assertEqual({"args": tuple(wrap_args), "kwargs": wrap_kwargs},
+                         validator())
+        self.assertEqual(wrap_validator, scenario)
+
+        # Default permission
+        validator, = self._get_scenario_validators(func, scenario)
+        self.assertEqual(validator.permission,
+                         validation.consts.EndpointPermission.USER)
+
+        # Custom permission
+        func.permission = "another_permission"
+        del scenario.validators
+        validator, = self._get_scenario_validators(func, scenario, reset=False)
+        self.assertEqual(validator.permission, "another_permission")
+
+        # Default result
+        func_success = lambda *a, **kv: None
+        validator, = self._get_scenario_validators(func_success, scenario)
+        self.assertTrue(validator().is_valid)
+
+        # Failure result
+        func_failure = lambda *a, **kv: failure
+        validator, = self._get_scenario_validators(func_failure, scenario)
+        self.assertFalse(validator().is_valid)
+
+    def test_required_services(self):
+        available_services = {
+            consts.ServiceType.IDENTITY: consts.Service.KEYSTONE,
+            consts.ServiceType.COMPUTE: consts.Service.NOVA,
+            consts.ServiceType.IMAGE: consts.Service.GLANCE}
+
+        clients = mock.Mock(
+            services=mock.Mock(return_value=available_services))
+
+        # Unwrap
+        required_services = lambda *services:\
+            validation.required_services(*services)(lambda: None)\
+            .validators.pop()(clients=clients)
+
+        # Services are available
+        result = required_services(consts.Service.KEYSTONE)
+        self.assertTrue(result.is_valid)
+
+        # Service is not available
+        service = consts.Service.CEILOMETER
+        result = required_services(consts.Service.KEYSTONE, service)
+        self.assertFalse(result.is_valid)
+
+        # Service is unknown
+        service = "unknown_service"
+        result = required_services(consts.Service.KEYSTONE, service)
+        self.assertFalse(result.is_valid)
+
     def test_add(self):
         def test_validator():
             pass

tests/test_osclients.py

@@ -17,6 +17,7 @@ from keystoneclient import exceptions as keystone_exceptions
 import mock
 from oslo.config import cfg
 
+from rally import consts
 from rally import exceptions
 from rally.objects import endpoint
 from rally import osclients
@@ -180,3 +181,18 @@ class OSClientsTestCase(test.TestCase):
         }
         mock_ironic.Client.assert_called_once_with("1.0", **kw)
         self.assertEqual(self.clients.cache["ironic"], fake_ironic)
+
+    @mock.patch("rally.osclients.Clients.keystone")
+    def test_services(self, mock_keystone):
+        available_services = {consts.ServiceType.IDENTITY: {},
+                              consts.ServiceType.COMPUTE: {},
+                              'unknown_service': {}
+                              }
+        mock_keystone.return_value = mock.Mock(service_catalog=mock.Mock(
+                get_endpoints=lambda: available_services))
+        clients = osclients.Clients({})
+
+        self.assertEqual(
+            clients.services(), {
+                consts.ServiceType.IDENTITY: consts.Service.KEYSTONE,
+                consts.ServiceType.COMPUTE: consts.Service.NOVA})