Add @validation.require_openstack() validator

This patch removes from @base.scenario(admin_only=False) argument.
That actually doesn't work and doesn't cover situation when we need
only users or admins or admins & users..

Add proper @require_openstack validator that actually checks
everything that is required.
So you can specify that only users or admin is required or both
users and admin are required.

Add missing "users" context in rally-scenario for scenarios that
use it.

Add missing "sla" sections in benchmarks

Reduce amount of iterations in "rps" benchmark

Change-Id: Ie92dbd473ee94c2073401c6da8dc2771c9dfaed8

rally-scenarios/rally.yaml

@@ -95,6 +95,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -112,6 +116,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -129,6 +137,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -146,6 +158,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -155,6 +171,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -164,6 +184,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -173,6 +197,10 @@
         type: "constant"
         times: 10
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -192,9 +220,8 @@
         sleep: 0.001
       runner:
         type: "rps"
-        times: 10000
+        times: 2000
         rps: 200
-        timeout: 600
       sla:
         max_failure_percent: 0
 
@@ -283,6 +310,12 @@
         type: "constant"
         times: 20
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+      sla:
+        max_failure_percent: 0
 
   CeilometerQueries.create_and_query_alarms:
     -
@@ -301,6 +334,10 @@
         type: "constant"
         times: 20
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -320,6 +357,10 @@
         type: "constant"
         times: 20
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
       sla:
         max_failure_percent: 0
 
@@ -338,6 +379,12 @@
         type: "constant"
         times: 20
         concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+      sla:
+        max_failure_percent: 0
 
   HeatStacks.create_and_list_stack:
     -
@@ -609,6 +656,8 @@
         users:
           tenants: 2
           users_per_tenant: 2
+      sla:
+        max_failure_percent: 0
 
   GlanceImages.create_and_delete_image:
     -

rally/benchmark/scenarios/authenticate/authenticate.py

@@ -22,11 +22,13 @@ class Authenticate(base.Scenario):
     For different types of clients like Keystone.
     """
 
+    @validation.required_openstack(users=True)
     @base.scenario()
     def keystone(self, **kwargs):
         self.clients("keystone")
 
-    @validation.required_parameters("repetitions")
+    @validation.number("repetitions", minval=1)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def validate_glance(self, repetitions):
         """Check Glance Client to ensure validation of token.
@@ -43,7 +45,8 @@ class Authenticate(base.Scenario):
             with base.AtomicAction(self, "authenticate.validate_glance"):
                 list(glance_client.images.list(name=image_name))
 
-    @validation.required_parameters("repetitions")
+    @validation.number("repetitions", minval=1)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def validate_nova(self, repetitions):
         """Check Nova Client to ensure validation of token.
@@ -58,7 +61,8 @@ class Authenticate(base.Scenario):
             with base.AtomicAction(self, "authenticate.validate_nova"):
                 nova_client.flavors.list()
 
-    @validation.required_parameters("repetitions")
+    @validation.number("repetitions", minval=1)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def validate_cinder(self, repetitions):
         """Check Cinder Client to ensure validation of token.
@@ -73,7 +77,8 @@ class Authenticate(base.Scenario):
             with base.AtomicAction(self, "authenticate.validate_cinder"):
                 cinder_client.volume_types.list()
 
-    @validation.required_parameters("repetitions")
+    @validation.number("repetitions", minval=1)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def validate_neutron(self, repetitions):
         """Check Neutron Client to ensure validation of token.
@@ -88,7 +93,8 @@ class Authenticate(base.Scenario):
             with base.AtomicAction(self, "authenticate.validate_neutron"):
                 neutron_client.get_auth_info()
 
-    @validation.required_parameters("repetitions")
+    @validation.number("repetitions", minval=1)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def validate_heat(self, repetitions):
         """Check Heat Client to ensure validation of token.

rally/benchmark/scenarios/base.py

@@ -25,17 +25,17 @@ from rally import exceptions
 from rally import utils
 
 
-def scenario(admin_only=False, context=None):
+def scenario(context=None):
-    """Add extra fields to benchmark scenarios methods.
+    """Make from plain python method benchmark.
 
-       This method is used as decorator for the methods of benchmark scenarios
+       It sets 2 attributes to function:
-       and it adds following extra fields to the methods.
+       is_scenario = True # that is used during discovering
-       'is_scenario' is set to True
+       func.context = context # default context for benchmark
-       'admin_only' is set to True if a scenario require admin endpoints
+
+       :param context: Default benchmark context
     """
     def wrapper(func):
         func.is_scenario = True
-        func.admin_only = admin_only
         func.context = context or {}
         return func
     return wrapper

rally/benchmark/scenarios/ceilometer/alarms.py

@@ -21,6 +21,7 @@ from rally import consts
 class CeilometerAlarms(ceilometerutils.CeilometerScenario):
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_alarm(self, meter_name, threshold, **kwargs):
         """Test creating an alarm.
@@ -36,6 +37,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._create_alarm(meter_name, threshold, kwargs)
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def list_alarms(self):
         """Test fetching all alarms.
@@ -45,6 +47,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._list_alarms()
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_list_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and getting newly created alarm.
@@ -62,6 +65,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._list_alarms(alarm.alarm_id)
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_update_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and updating the newly created alarm.
@@ -80,6 +84,7 @@ class CeilometerAlarms(ceilometerutils.CeilometerScenario):
         self._update_alarm(alarm.alarm_id, alarm_dict_diff)
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_delete_alarm(self, meter_name, threshold, **kwargs):
         """Test creating and deleting the newly created alarm.

rally/benchmark/scenarios/ceilometer/meters.py

@@ -21,6 +21,7 @@ from rally import consts
 class CeilometerMeters(ceilometerutils.CeilometerScenario):
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def list_meters(self):
         """Test fetching user's meters."""

rally/benchmark/scenarios/ceilometer/queries.py

@@ -23,6 +23,7 @@ from rally import consts
 class CeilometerQueries(ceilometerutils.CeilometerScenario):
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_query_alarms(self, meter_name, threshold, filter=None,
                                 orderby=None, limit=None, **kwargs):
@@ -44,6 +45,7 @@ class CeilometerQueries(ceilometerutils.CeilometerScenario):
         self._query_alarms(filter, orderby, limit)
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_query_alarm_history(self, meter_name, threshold,
                                        orderby=None, limit=None, **kwargs):
@@ -63,6 +65,7 @@ class CeilometerQueries(ceilometerutils.CeilometerScenario):
         self._query_alarm_history(alarm_filter, orderby, limit)
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_and_query_samples(self, counter_name, counter_type,
                                  counter_unit, counter_volume, resource_id,

rally/benchmark/scenarios/ceilometer/resources.py

@@ -21,6 +21,7 @@ from rally import consts
 class CeilometerResource(ceilometerutils.CeilometerScenario):
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario()
     def list_resources(self):
         """Test fetching all resources.

rally/benchmark/scenarios/ceilometer/stats.py

@@ -21,6 +21,7 @@ from rally import consts
 class CeilometerStats(utils.CeilometerScenario):
 
     @validation.required_services(consts.Service.CEILOMETER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["ceilometer"]})
     def create_meter_and_get_stats(self, **kwargs):
         """Test creating a meter and fetching its statistics.

rally/benchmark/scenarios/cinder/volumes.py

@@ -25,6 +25,7 @@ class CinderVolumes(utils.CinderScenario,
                     nova_utils.NovaScenario):
 
     @validation.required_services(consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["cinder"]})
     def create_and_list_volume(self, size, detailed=True, **kwargs):
         """Tests creating a volume and listing volumes.
@@ -43,6 +44,7 @@ class CinderVolumes(utils.CinderScenario,
         self._list_volumes(detailed)
 
     @validation.required_services(consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["cinder"]})
     def create_and_delete_volume(self, size, min_sleep=0, max_sleep=0,
                                  **kwargs):
@@ -56,6 +58,7 @@ class CinderVolumes(utils.CinderScenario,
         self._delete_volume(volume)
 
     @validation.required_services(consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["cinder"]})
     def create_volume(self, size, **kwargs):
         """Test creating volumes perfromance.
@@ -65,9 +68,10 @@ class CinderVolumes(utils.CinderScenario,
         """
         self._create_volume(size, **kwargs)
 
-    @base.scenario(context={"cleanup": ["cinder"]})
     @validation.required_services(consts.Service.CINDER)
     @validation.required_contexts("volumes")
+    @validation.required_openstack(users=True)
+    @base.scenario(context={"cleanup": ["cinder"]})
     def create_and_delete_snapshot(self, force=False, min_sleep=0,
                                    max_sleep=0, **kwargs):
         """Tests creating and then deleting a volume-snapshot."""
@@ -85,6 +89,7 @@ class CinderVolumes(utils.CinderScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA, consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["cinder", "nova"]})
     def create_and_attach_volume(self, volume_size, image, flavor,
                                  min_sleep=0, max_sleep=0, **kwargs):

rally/benchmark/scenarios/designate/basic.py

@@ -23,6 +23,7 @@ from rally import consts
 class DesignateBasic(utils.DesignateScenario):
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def create_and_list_domains(self):
         """Tests creating a domain and listing domains.
@@ -40,6 +41,7 @@ class DesignateBasic(utils.DesignateScenario):
         self._list_domains()
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def list_domains(self):
         """Test the designate domain-list command.
@@ -55,6 +57,7 @@ class DesignateBasic(utils.DesignateScenario):
         self._list_domains()
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def create_and_delete_domain(self):
         """Test adds and then deletes domain.
@@ -66,6 +69,7 @@ class DesignateBasic(utils.DesignateScenario):
         self._delete_domain(domain['id'])
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def create_and_delete_records(self, records_per_domain=5):
         """Test adds and then deletes records.
@@ -92,6 +96,7 @@ class DesignateBasic(utils.DesignateScenario):
                     domain['id'], record['id'], atomic_action=False)
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def list_records(self, domain_id):
         """Test the designate record-list command.
@@ -109,6 +114,7 @@ class DesignateBasic(utils.DesignateScenario):
         self._list_records(domain_id)
 
     @validation.required_services(consts.Service.DESIGNATE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["designate"]})
     def create_and_list_records(self, records_per_domain=5):
         """Test adds and then lists records.

rally/benchmark/scenarios/glance/images.py

@@ -27,6 +27,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
     RESOURCE_NAME_LENGTH = 16
 
     @validation.required_services(consts.Service.GLANCE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["glance"]})
     def create_and_list_image(self, container_format,
                               image_location, disk_format, **kwargs):
@@ -49,6 +50,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
         self._list_images()
 
     @validation.required_services(consts.Service.GLANCE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["glance"]})
     def list_images(self):
         """Test the glance image-list command.
@@ -64,6 +66,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
         self._list_images()
 
     @validation.required_services(consts.Service.GLANCE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["glance"]})
     def create_and_delete_image(self, container_format,
                                 image_location, disk_format, **kwargs):
@@ -79,6 +82,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
     @types.set(flavor=types.FlavorResourceType)
     @validation.flavor_exists("flavor")
     @validation.required_services(consts.Service.GLANCE, consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["glance", "nova"]})
     def create_image_and_boot_instances(self, container_format,
                                         image_location, disk_format,

rally/benchmark/scenarios/heat/stacks.py

@@ -36,6 +36,7 @@ class HeatStacks(utils.HeatScenario):
         return template
 
     @validation.required_services(consts.Service.HEAT)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"],
                             "roles": ["heat_stack_owner"]})
     def create_and_list_stack(self, template_path=None):
@@ -55,6 +56,7 @@ class HeatStacks(utils.HeatScenario):
         self._list_stacks()
 
     @validation.required_services(consts.Service.HEAT)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"],
                             "roles": ["heat_stack_owner"]})
     def create_and_delete_stack(self, template_path=None):

rally/benchmark/scenarios/keystone/basic.py

@@ -20,33 +20,45 @@ from rally.benchmark import validation
 
 class KeystoneBasic(kutils.KeystoneScenario):
 
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("name_length", minval=10)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_user(self, name_length=10, **kwargs):
         self._user_create(name_length=name_length, **kwargs)
 
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("name_length", minval=10)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_delete_user(self, name_length=10, **kwargs):
         user = self._user_create(name_length=name_length, **kwargs)
         self._resource_delete(user)
 
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("name_length", minval=10)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_tenant(self, name_length=10, **kwargs):
         self._tenant_create(name_length=name_length, **kwargs)
 
-    @validation.required_parameters("users_per_tenant")
+    @validation.number("name_length", minval=10)
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("users_per_tenant", minval=1)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_tenant_with_users(self, users_per_tenant, name_length=10,
                                  **kwargs):
         tenant = self._tenant_create(name_length=name_length, **kwargs)
         self._users_create(tenant, users_per_tenant=users_per_tenant,
                            name_length=name_length)
 
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("name_length", minval=10)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_and_list_users(self, name_length=10, **kwargs):
         self._user_create(name_length=name_length, **kwargs)
         self._list_users()
 
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["keystone"]})
+    @validation.number("name_length", minval=10)
+    @validation.required_openstack(admin=True)
+    @base.scenario(context={"admin_cleanup": ["keystone"]})
     def create_and_list_tenants(self, name_length=10, **kwargs):
         self._tenant_create(name_length=name_length, **kwargs)
         self._list_tenants()

rally/benchmark/scenarios/neutron/network.py

@@ -22,6 +22,7 @@ from rally import consts
 class NeutronNetworks(utils.NeutronScenario):
 
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_list_networks(self, network_create_args=None):
         """Create a network and then listing all networks.
@@ -41,6 +42,7 @@ class NeutronNetworks(utils.NeutronScenario):
         self._list_networks()
 
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_delete_networks(self, network_create_args=None):
         """Create a network and then deleting it.
@@ -53,8 +55,9 @@ class NeutronNetworks(utils.NeutronScenario):
         network = self._create_network(network_create_args or {})
         self._delete_network(network['network'])
 
-    @validation.required_parameters("subnets_per_network")
+    @validation.number("subnets_per_network", minval=1, integer_only=True)
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_list_subnets(self,
                                 network_create_args=None,
@@ -80,8 +83,9 @@ class NeutronNetworks(utils.NeutronScenario):
 
         self._list_subnets()
 
-    @validation.required_parameters("subnets_per_network")
+    @validation.number("subnets_per_network", minval=1, integer_only=True)
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_delete_subnets(self,
                                   network_create_args=None,
@@ -107,8 +111,9 @@ class NeutronNetworks(utils.NeutronScenario):
                                          subnet_create_args or {})
             self._delete_subnet(subnet)
 
-    @validation.required_parameters("subnets_per_network")
+    @validation.number("subnets_per_network", minval=1, integer_only=True)
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_list_routers(self,
                                 network_create_args=None,
@@ -140,8 +145,9 @@ class NeutronNetworks(utils.NeutronScenario):
 
         self._list_routers()
 
-    @validation.required_parameters("ports_per_network")
+    @validation.number("ports_per_network", minval=1, integer_only=True)
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_list_ports(self,
                               network_create_args=None,
@@ -159,8 +165,9 @@ class NeutronNetworks(utils.NeutronScenario):
 
         self._list_ports()
 
-    @validation.required_parameters("ports_per_network")
+    @validation.number("ports_per_network", minval=1, integer_only=True)
     @validation.required_services(consts.Service.NEUTRON)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["neutron"]})
     def create_and_delete_ports(self,
                                 network_create_args=None,

rally/benchmark/scenarios/nova/servers.py

@@ -39,6 +39,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova"]})
     def boot_and_list_server(self, image, flavor,
                              detailed=True, **kwargs):
@@ -61,6 +62,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova"]})
     def boot_and_delete_server(self, image, flavor,
                                min_sleep=0, max_sleep=0, **kwargs):
@@ -74,6 +76,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA, consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova", "cinder"]})
     def boot_server_from_volume_and_delete(self, image, flavor,
                                            volume_size,
@@ -92,6 +95,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova"]})
     def boot_and_bounce_server(self, image, flavor, **kwargs):
         """Test booting a server with further performing specified actions.
@@ -118,6 +122,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA, consts.Service.GLANCE)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova", "glance"]})
     def snapshot_server(self, image, flavor, **kwargs):
         """Tests Nova instance snapshotting."""
@@ -135,6 +140,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova"]})
     def boot_server(self, image, flavor, auto_assign_nic=False, **kwargs):
         """Test VM boot - assumed clean-up is done elsewhere."""
@@ -146,6 +152,7 @@ class NovaServers(utils.NovaScenario,
                flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA, consts.Service.CINDER)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova", "cinder"]})
     def boot_server_from_volume(self, image, flavor, volume_size,
                                 auto_assign_nic=False, **kwargs):
@@ -203,6 +210,7 @@ class NovaServers(utils.NovaScenario,
                to_flavor=types.FlavorResourceType)
     @validation.image_valid_on_flavor("flavor", "image")
     @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["nova"]})
     def resize_server(self, image, flavor, to_flavor, **kwargs):
         """Tests resize serveri."""

rally/benchmark/scenarios/quotas/quotas.py

@@ -22,7 +22,8 @@ from rally import consts
 class Quotas(utils.QuotasScenario):
 
     @validation.required_services(consts.Service.NOVA)
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["quotas"]})
+    @validation.required_openstack(admin=True, users=True)
+    @base.scenario(context={"admin_cleanup": ["quotas"]})
     def nova_update(self, max_quota=1024):
         """Tests updating quotas for nova.
 
@@ -32,7 +33,8 @@ class Quotas(utils.QuotasScenario):
         self._update_quotas('nova', tenant_id, max_quota)
 
     @validation.required_services(consts.Service.NOVA)
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["quotas"]})
+    @validation.required_openstack(admin=True, users=True)
+    @base.scenario(context={"admin_cleanup": ["quotas"]})
     def nova_update_and_delete(self, max_quota=1024):
         """Tests updating and deleting quotas for nova.
 
@@ -44,7 +46,8 @@ class Quotas(utils.QuotasScenario):
         self._delete_quotas('nova', tenant_id)
 
     @validation.required_services(consts.Service.CINDER)
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["quotas"]})
+    @validation.required_openstack(admin=True, users=True)
+    @base.scenario(context={"admin_cleanup": ["quotas"]})
     def cinder_update(self, max_quota=1024):
         """Tests updating quotas for cinder.
 
@@ -54,7 +57,8 @@ class Quotas(utils.QuotasScenario):
         self._update_quotas('cinder', tenant_id, max_quota)
 
     @validation.required_services(consts.Service.CINDER)
-    @base.scenario(admin_only=True, context={"admin_cleanup": ["quotas"]})
+    @validation.required_openstack(admin=True, users=True)
+    @base.scenario(context={"admin_cleanup": ["quotas"]})
     def cinder_update_and_delete(self, max_quota=1024):
         """Tests updating and deleting quotas for cinder.
 

rally/benchmark/scenarios/requests/http_requests.py

@@ -12,7 +12,7 @@
 
 import requests
 
-from rally.benchmark.scenarios import base as scenario_base
+from rally.benchmark.scenarios import base
 from rally import exceptions
 from rally.openstack.common.gettextutils import _
 
@@ -21,10 +21,10 @@ class WrongStatusException(exceptions.RallyException):
     msg_fmt = _("Requests scenario exception: '%(message)s'")
 
 
-class Requests(scenario_base.Scenario):
+class Requests(base.Scenario):
     """This class should contain all the http_request scenarios."""
 
-    @scenario_base.scenario()
+    @base.scenario()
     def check_response(self, url, response=None):
         """Standard way to benchmark web services.
 

rally/benchmark/scenarios/sahara/clusters.py

@@ -27,9 +27,10 @@ class SaharaClusters(utils.SaharaScenario):
 
     @types.set(flavor=types.FlavorResourceType)
     @validation.flavor_exists('flavor')
-    @validation.required_services(consts.Service.SAHARA)
     @validation.required_contexts("users", "sahara_image")
     @validation.number("node_count", minval=2, integer_only=True)
+    @validation.required_services(consts.Service.SAHARA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["sahara"]})
     def create_and_delete_cluster(self, flavor, node_count, plugin_name,
                                   hadoop_version, floating_ip_pool=None,

rally/benchmark/scenarios/sahara/node_group_templates.py

@@ -25,6 +25,7 @@ class SaharaNodeGroupTemplates(utils.SaharaScenario):
     @types.set(flavor=types.FlavorResourceType)
     @validation.flavor_exists('flavor')
     @validation.required_services(consts.Service.SAHARA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["sahara"]})
     def create_and_list_node_group_templates(self, flavor,
                                              plugin_name="vanilla",
@@ -59,6 +60,7 @@ class SaharaNodeGroupTemplates(utils.SaharaScenario):
     @types.set(flavor=types.FlavorResourceType)
     @validation.flavor_exists('flavor')
     @validation.required_services(consts.Service.SAHARA)
+    @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["sahara"]})
     def create_delete_node_group_templates(self, flavor,
                                            plugin_name="vanilla",

rally/benchmark/scenarios/tempest/tempest.py

@@ -22,6 +22,7 @@ from rally import consts
 class TempestScenario(base.Scenario):
 
     @validation.tempest_tests_exists()
+    @validation.required_openstack(admin=True)
     @base.scenario(context={"tempest": {}})
     @utils.tempest_log_wrapper
     def single_test(self, test_name, log_file):
@@ -36,6 +37,7 @@ class TempestScenario(base.Scenario):
 
         self.context()["verifier"].run(test_name, log_file)
 
+    @validation.required_openstack(admin=True)
     @base.scenario(context={"tempest": {}})
     @utils.tempest_log_wrapper
     def all(self, log_file):
@@ -47,6 +49,7 @@ class TempestScenario(base.Scenario):
         self.context()["verifier"].run("", log_file)
 
     @validation.tempest_set_exists()
+    @validation.required_openstack(admin=True)
     @base.scenario(context={"tempest": {}})
     @utils.tempest_log_wrapper
     def set(self, set_name, log_file):
@@ -66,6 +69,7 @@ class TempestScenario(base.Scenario):
         self._context["verifier"].run(testr_arg, log_file)
 
     @validation.tempest_tests_exists()
+    @validation.required_openstack(admin=True)
     @base.scenario(context={"tempest": {}})
     @utils.tempest_log_wrapper
     def list_of_tests(self, test_names, log_file):
@@ -77,6 +81,7 @@ class TempestScenario(base.Scenario):
 
         self._context["verifier"].run(" ".join(test_names), log_file)
 
+    @validation.required_openstack(admin=True)
     @base.scenario(context={"tempest": {}})
     @utils.tempest_log_wrapper
     def specific_regex(self, regex, log_file):

rally/benchmark/scenarios/vm/vmtasks.py

@@ -36,9 +36,10 @@ class VMTasks(nova_utils.NovaScenario, vm_utils.VMScenario):
     @validation.number("port", minval=1, maxval=65535, nullable=True,
                        integer_only=True)
     @validation.external_network_exists("floating_network", "use_floatingip")
+    @validation.required_services(consts.Service.NOVA)
+    @validation.required_openstack(users=True)
     @base.scenario(
         context={"cleanup": ["nova"], "keypair": {}, "allow_ssh": {}})
-    @validation.required_services(consts.Service.NOVA)
     def boot_runcommand_delete(self, image, flavor,
                                script, interpreter, username,
                                fixed_network="private",

rally/benchmark/validation.py

@@ -22,6 +22,7 @@ from novaclient import exceptions as nova_exc
 from rally.benchmark import types as types
 from rally import consts
 from rally import exceptions
+from rally import objects
 from rally.openstack.common.gettextutils import _
 from rally.verification.verifiers.tempest import tempest
 
@@ -376,3 +377,38 @@ def required_contexts(config, clients, task, *context_names):
         return ValidationResult(False, message)
     else:
         return ValidationResult()
+
+
+@validator
+def required_openstack(config, clients, task, admin=False, users=False):
+    """Validator that requires OpenStack admin or (and) users.
+
+    This allows us to create 4 kind of benchmarks:
+    1) not OpenStack related (validator is not specified)
+    2) requires OpenStack admin
+    3) requires OpenStack admin + users
+    4) requires OpenStack users
+
+    :param admin: requires OpenStack admin
+    :param users: requires OpenStack users
+    """
+
+    if not (admin or users):
+        return ValidationResult(
+            False, _("You should specify admin=True or users=True or both."))
+
+    deployment = objects.Deployment.get(task["deployment_uuid"])
+
+    if deployment["admin"] and deployment["users"]:
+        return ValidationResult()
+
+    if deployment["admin"]:
+        if users and not config.get("context", {}).get("users"):
+            return ValidationResult(False,
+                                    _("You should specify 'users' context"))
+        return ValidationResult()
+
+    if deployment["users"] and admin:
+        return ValidationResult(False, _("Admin credentials required"))
+
+    return ValidationResult()

tests/benchmark/test_validation.py

@@ -385,3 +385,83 @@ class ValidatorsTestCase(test.TestCase):
         result = validator({"context": {"c1": 1, "c2": 2, "c3": 3, "a": 1}},
                            None, None)
         self.assertTrue(result.is_valid, result.msg)
+
+    @mock.patch("rally.benchmark.validation.objects.Deployment.get")
+    def test_required_openstack_with_admin(self, mock_deploy_get):
+        validator = self._unwrap_validator(validation.required_openstack,
+                                           admin=True)
+
+        # admin presented in deployment
+        task = {"deployment_uuid": mock.MagicMock()}
+        mock_deploy_get.return_value = {"admin": "admin_endpoint", "users": []}
+        self.assertTrue(validator(None, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        # admin not presented in deployment
+        mock_deploy_get.return_value = {"admin": None, "users": ["u1", "h2"]}
+        self.assertFalse(validator(None, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+    @mock.patch("rally.benchmark.validation.objects.Deployment.get")
+    def test_required_openstack_with_users(self, mock_deploy_get):
+        validator = self._unwrap_validator(validation.required_openstack,
+                                           users=True)
+
+        # users presented in deployment
+        task = {"deployment_uuid": mock.MagicMock()}
+        mock_deploy_get.return_value = {"admin": None, "users": ["u_endpoint"]}
+        self.assertTrue(validator({}, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        # admin and users presented in deployment
+        mock_deploy_get.return_value = {"admin": "a", "users": ["u1", "h2"]}
+        self.assertTrue(validator({}, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        # admin and user context
+        mock_deploy_get.return_value = {"admin": "a", "users": []}
+        context = {"context": {"users": True}}
+        self.assertTrue(validator(context, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        # just admin presented
+        mock_deploy_get.return_value = {"admin": "a", "users": []}
+        self.assertFalse(validator({}, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+    @mock.patch("rally.benchmark.validation.objects.Deployment.get")
+    def test_required_openstack_with_admin_and_users(self, mock_deploy_get):
+        validator = self._unwrap_validator(validation.required_openstack,
+                                           admin=True, users=True)
+
+        task = {"deployment_uuid": mock.MagicMock()}
+
+        mock_deploy_get.return_value = {"admin": "a", "users": []}
+        self.assertFalse(validator({}, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        mock_deploy_get.return_value = {"admin": "a", "users": ["u"]}
+        self.assertTrue(validator({}, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+        mock_deploy_get.reset_mock()
+
+        # admin and user context
+        mock_deploy_get.return_value = {"admin": "a", "users": []}
+        context = {"context": {"users": True}}
+        self.assertTrue(validator(context, None, task).is_valid)
+        mock_deploy_get.assert_called_once_with(task["deployment_uuid"])
+
+    def test_required_openstack_invalid(self):
+        validator = self._unwrap_validator(validation.required_openstack)
+        self.assertFalse(validator(None, None, None).is_valid)