Reworked script generating tempest.conf

* The config.py script generating tempest.conf was significantly reworked.

* The config.ini file was updated.

* A context class to create/delete all resources needed for Tempest was added.
  How does it work? Before running tests:
    1. The context class creates all roles needed for Tempest;
    2. If the image_ref option is not specified in the tempest.conf file,
       the context class will create an image for this option and add ID of
       the created image to tempest.conf;
    3. The same is for 'image_ref_alt', 'flavor_ref' and 'flavor_ref_alt';
    4. Once tests finish, all created resources will be deleted. All IDs of
       created resources will be deleted from tempest.conf.

* The TempestConfigCreationFailure exception was moved to the exceptions.py
  file. In addition, the TempestResourceCreationFailure exception was added.

* The rally-verify.sh script was changed. The index_verify.mako file was changed
  as well in accordance with changes in rally-verify.sh.

* All relevant unit tests were updated in accordance with new changes.

Closes-bug: #1454648

Change-Id: If32e51d6ce1f6bf7421021d151005cc75ed34953

etc/rally/rally.conf.sample

@@ -615,14 +615,29 @@
 # From rally
 #
 
-# Version of cirros image (string value)
+# CirrOS image URL (string value)
-#cirros_version = 0.3.4
+#cirros_img_url = http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img
 
-# Cirros image name (string value)
-#cirros_image = cirros-0.3.4-x86_64-disk.img
 
-# Cirros image base URL (string value)
+[role]
-#cirros_base_url = http://download.cirros-cloud.net
+
+#
+# From rally
+#
+
+# Role required for users to be able to create Swift containers
+# (string value)
+#swift_operator_role = Member
+
+# User role that has reseller admin (string value)
+#swift_reseller_admin_role = ResellerAdmin
+
+# Role required for users to be able to manage Heat stacks (string
+# value)
+#heat_stack_owner_role = heat_stack_owner
+
+# Role for Heat template-defined users (string value)
+#heat_stack_user_role = heat_stack_user
 
 
 [users_context]

rally/common/opts.py

@@ -43,6 +43,7 @@ def list_opts():
                          ec2_utils.EC2_BENCHMARK_OPTS)),
         ("image",
          itertools.chain(tempest_conf.IMAGE_OPTS)),
+        ("role", itertools.chain(tempest_conf.ROLE_OPTS)),
         ("users_context", itertools.chain(users.USER_CONTEXT_OPTS)),
         ("cleanup", itertools.chain(cleanup_base.CLEANUP_OPTS))
     ]

rally/exceptions.py

@@ -231,3 +231,11 @@ class InvalidHostException(RallyException):
 
 class MultipleMatchesFound(RallyException):
     msg_fmt = _("Found multiple %(needle)s: %(haystack)s")
+
+
+class TempestConfigCreationFailure(RallyException):
+    msg_fmt = _("Unable to create Tempest config file: %(message)s")
+
+
+class TempestResourceCreationFailure(RallyException):
+    msg_fmt = _("Unable to create resource needed for Tempest: %(message)s")

rally/plugins/openstack/context/not_for_production/tempest.py

@@ -23,7 +23,6 @@ from rally.common import utils
 from rally import consts
 from rally import exceptions
 from rally.task import context
-from rally.verification.tempest import config
 from rally.verification.tempest import tempest
 
 LOG = logging.getLogger(__name__)
@@ -65,7 +64,7 @@ class Tempest(context.Context):
             msg = _("Failing to install tempest.")
             LOG.error(msg)
             raise exceptions.BenchmarkSetupFailure(msg)
-        except config.TempestConfigCreationFailure:
+        except exceptions.TempestConfigCreationFailure:
             msg = _("Failing to configure tempest.")
             LOG.error(msg)
             raise exceptions.BenchmarkSetupFailure(msg)

rally/verification/tempest/config.ini

@@ -3,50 +3,55 @@ debug = True
 log_file = tempest.log
 use_stderr = False
 
-[auth]
-allow_tenant_isolation = False
-
 [boto]
-instance_type = m1.nano
 http_socket_timeout = 30
-build_interval = 1
-build_timeout = 196
 
 [compute]
-ssh_connect_method = floating
+image_ref =
+image_ref_alt =
+flavor_ref =
+flavor_ref_alt =
+ssh_user = cirros
 image_ssh_user = cirros
 image_alt_ssh_user = cirros
-network_for_ssh = private
-ssh_user = cirros
-build_interval = 1
-build_timeout = 196
 
 [compute-feature-enabled]
-change_password = False
 live_migration = False
-block_migration_for_live_migration = False
+resize = True
+vnc_console = False
+attach_encrypted_volume = False
+
+[dashboard]
+
+[data_processing-feature-enabled]
+plugins = vanilla,cdh,mapr,spark,ambari
 
 [identity]
 
+[image-feature-enabled]
+deactivate_image = True
+
 [network]
-tenant_networks_reachable = False
 
 [network-feature-enabled]
-api_extensions = all
+ipv6_subnet_attributes = True
 ipv6 = True
 
+[object-storage]
+
 [oslo_concurrency]
 
+[orchestration]
+max_template_size = 5440000
+max_resources_per_stack = 20000
+
 [scenario]
-large_ops_number = 0
+large_ops_number = 2
-ssh_user = cirros
 
 [service_available]
 
 [validation]
-ssh_timeout = 196
+run_validation = False
-ip_version_for_ssh = 4
 
-[volume]
+[volume-feature-enabled]
-build_interval = 1
+bootable = True
-build_timeout = 196

rally/verification/tempest/config.py

@@ -13,10 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import datetime
 import inspect
 import os
-import time
+import uuid
 
 from oslo_config import cfg
 import requests
@@ -30,236 +29,328 @@ from rally.common import objects
 from rally import exceptions
 from rally import osclients
 
-
 LOG = logging.getLogger(__name__)
 
-
 IMAGE_OPTS = [
-    cfg.StrOpt("cirros_version",
+    cfg.StrOpt("cirros_img_url",
-               default="0.3.4",
+               default="http://download.cirros-cloud.net/"
-               help="Version of cirros image"),
+                       "0.3.4/cirros-0.3.4-x86_64-disk.img",
-    cfg.StrOpt("cirros_image",
+               help="CirrOS image URL")
-               default="cirros-0.3.4-x86_64-disk.img",
-               help="Cirros image name"),
-    cfg.StrOpt("cirros_base_url",
-               default="http://download.cirros-cloud.net",
-               help="Cirros image base URL"),
 ]
+
+ROLE_OPTS = [
+    cfg.StrOpt("swift_operator_role",
+               default="Member",
+               help="Role required for users "
+                    "to be able to create Swift containers"),
+    cfg.StrOpt("swift_reseller_admin_role",
+               default="ResellerAdmin",
+               help="User role that has reseller admin"),
+    cfg.StrOpt("heat_stack_owner_role",
+               default="heat_stack_owner",
+               help="Role required for users "
+                    "to be able to manage Heat stacks"),
+    cfg.StrOpt("heat_stack_user_role",
+               default="heat_stack_user",
+               help="Role for Heat template-defined users")
+]
+
 CONF = cfg.CONF
 CONF.register_opts(IMAGE_OPTS, "image")
+CONF.register_opts(ROLE_OPTS, "role")
+
+IMAGE_NAME = parse.urlparse(CONF.image.cirros_img_url).path.split("/")[-1]
 
 
-class TempestConfigCreationFailure(exceptions.RallyException):
+def _create_or_get_data_dir():
-    msg_fmt = _("Unable create tempest.conf: '%(message)s'")
+    data_dir = os.path.join(
+        os.path.expanduser("~"), ".rally", "tempest", "data")
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir)
+
+    return data_dir
 
 
-class TempestConf(object):
+def _write_config(conf_path, conf_data):
+    with open(conf_path, "w+") as conf_file:
+        conf_data.write(conf_file)
+
+
+class TempestConfig(object):
+    """Class to generate Tempest configuration file."""
 
     def __init__(self, deployment):
+        self.deployment = deployment
+
         self.endpoint = db.deployment_get(deployment)["admin"]
         self.clients = osclients.Clients(objects.Endpoint(**self.endpoint))
-        try:
+        self.keystone = self.clients.verified_keystone()
-            self.keystoneclient = self.clients.verified_keystone()
-        except exceptions.InvalidAdminException:
-            msg = (_("Admin permission is required to generate tempest "
-                     "configuration file. User %s doesn't have admin role.") %
-                   self.endpoint["username"])
-            raise TempestConfigCreationFailure(msg)
 
         self.available_services = self.clients.services().values()
 
+        self.data_dir = _create_or_get_data_dir()
+
         self.conf = configparser.ConfigParser()
         self.conf.read(os.path.join(os.path.dirname(__file__), "config.ini"))
-        self.deployment = deployment
-        self.data_path = os.path.join(os.path.expanduser("~"), ".rally",
-                                      "tempest", "data")
-        if not os.path.exists(self.data_path):
-            os.makedirs(self.data_path)
-        self.img_path = os.path.join(self.data_path,
-                                     CONF.image.cirros_image)
-        if not os.path.isfile(self.img_path):
-            self._load_img()
 
-    def _load_img(self):
+        self._download_cirros_image()
-        cirros_url = ("%s/%s/%s" %
+
-                      (CONF.image.cirros_base_url,
+    def _download_cirros_image(self):
-                       CONF.image.cirros_version,
+        img_path = os.path.join(self.data_dir, IMAGE_NAME)
-                       CONF.image.cirros_image))
+        if os.path.isfile(img_path):
+            return
+
         try:
-            response = requests.get(cirros_url, stream=True)
+            response = requests.get(CONF.image.cirros_img_url, stream=True)
         except requests.ConnectionError as err:
-            msg = _("Error on downloading cirros image, possibly"
+            msg = _("Failed to download CirrOS image. "
-                    " no connection to Internet with message %s") % str(err)
+                    "Possibly there is no connection to Internet. "
-            raise TempestConfigCreationFailure(msg)
+                    "Error: %s.") % (str(err) or "unknown")
+            raise exceptions.TempestConfigCreationFailure(msg)
+
         if response.status_code == 200:
-            with open(self.img_path + ".tmp", "wb") as img_file:
+            with open(img_path + ".tmp", "wb") as img_file:
                 for chunk in response.iter_content(chunk_size=1024):
                     if chunk:   # filter out keep-alive new chunks
                         img_file.write(chunk)
                         img_file.flush()
-            os.rename(self.img_path + ".tmp", self.img_path)
+            os.rename(img_path + ".tmp", img_path)
         else:
             if response.status_code == 404:
-                msg = _("Error on downloading cirros image, possibly"
+                msg = _("Failed to download CirrOS image. "
-                        "invalid cirros_version or cirros_image in rally.conf")
+                        "Image was not found.")
             else:
-                msg = _("Error on downloading cirros image, "
+                msg = _("Failed to download CirrOS image. "
-                        "HTTP error code %s") % response.getcode()
+                        "HTTP error code %d.") % response.status_code
-            raise TempestConfigCreationFailure(msg)
+            raise exceptions.TempestConfigCreationFailure(msg)
 
-    def _get_url(self, servicename):
+    def _get_service_url(self, service_type):
-        services_type2name_map = self.clients.services()
+        for service in self.keystone.auth_ref["serviceCatalog"]:
-        for service in self.keystoneclient.auth_ref["serviceCatalog"]:
+            if self.clients.services().get(service["type"]) == service_type:
-            if services_type2name_map.get(service["type"]) == servicename:
                 return service["endpoints"][0]["publicURL"]
 
-    def _set_default(self):
+    def _configure_boto(self, section_name="boto"):
-        # Nothing to set up in this section for now
+        self.conf.set(section_name, "ec2_url", self._get_service_url("ec2"))
-        pass
+        self.conf.set(section_name, "s3_url", self._get_service_url("s3"))
-
+        self.conf.set(section_name, "s3_materials_path",
-    def _set_oslo_concurrency(self, section_name="oslo_concurrency"):
+                      os.path.join(self.data_dir, "s3materials"))
-        lock_path = os.path.join(self.data_path,
-                                 "lock_files_%s" % self.deployment)
-        if not os.path.exists(lock_path):
-            os.makedirs(lock_path)
-        self.conf.set(section_name, "lock_path", lock_path)
-
-    def _set_boto(self, section_name="boto"):
-        self.conf.set(section_name, "ec2_url", self._get_url("ec2"))
-        self.conf.set(section_name, "s3_url", self._get_url("s3"))
-        materials_path = os.path.join(self.data_path, "s3materials")
-        self.conf.set(section_name, "s3_materials_path", materials_path)
         # TODO(olkonami): find out how can we get ami, ari, aki manifest files
 
-    def _set_compute_images(self, section_name="compute"):
+    def _configure_default(self, section_name="DEFAULT"):
-        glanceclient = self.clients.glance()
+        # Nothing to configure in this section for now
-        image_list = [img for img in glanceclient.images.list()
+        pass
-                      if img.status.lower() == "active" and
-                      img.name is not None and "cirros" in img.name]
-        # Upload new images if there are no
-        # necessary images in the cloud (cirros)
-        while len(image_list) < 2:
-            now = (datetime.datetime.fromtimestamp(time.time()).
-                   strftime("%Y_%m_%d_%H_%M_%S"))
-            try:
-                image = glanceclient.images.create(name=("cirros_%s" % now),
-                                                   disk_format="qcow2",
-                                                   container_format="bare")
-                image.update(data=open(self.img_path, "rb"))
-                image_list.append(image)
-            except Exception as e:
-                msg = _("There are no desired images (cirros) or only one and "
-                        "new image could not be created.\n"
-                        "Reason: %s") % getattr(e, "message", "unknown")
-                raise TempestConfigCreationFailure(msg)
-        self.conf.set(section_name, "image_ref", image_list[0].id)
-        self.conf.set(section_name, "image_ref_alt", image_list[1].id)
 
-    def _set_compute_flavors(self, section_name="compute"):
+    def _configure_dashboard(self, section_name="dashboard"):
-        novaclient = self.clients.nova()
+        url = "http://%s/" % parse.urlparse(self.endpoint["auth_url"]).hostname
-        flavor_list = sorted(novaclient.flavors.list(),
+        self.conf.set(section_name, "dashboard_url", url)
-                             key=lambda flv: flv.ram)
-        # Create new flavors if they are missing
-        while len(flavor_list) < 2:
-            now = (datetime.datetime.fromtimestamp(time.time()).
-                   strftime("%Y_%m_%d_%H_%M_%S"))
-            try:
-                flv = novaclient.flavors.create("m1.tiny_%s" % now, 512, 1, 1)
-                flavor_list.append(flv)
-            except Exception as e:
-                msg = _("There are no desired flavors or only one and "
-                        "new flavor could not be created.\n"
-                        "Reason: %s") % getattr(e, "message", "unknown")
-                raise TempestConfigCreationFailure(msg)
-        self.conf.set(section_name, "flavor_ref", flavor_list[0].id)
-        self.conf.set(section_name, "flavor_ref_alt", flavor_list[1].id)
 
-    def _set_compute_ssh_connect_method(self, section_name="compute"):
+    def _configure_identity(self, section_name="identity"):
-        if "neutron" in self.available_services:
-            self.conf.set(section_name, "ssh_connect_method", "floating")
-        else:
-            self.conf.set(section_name, "ssh_connect_method", "fixed")
-
-    def _set_identity(self, section_name="identity"):
         self.conf.set(section_name, "username", self.endpoint["username"])
         self.conf.set(section_name, "password", self.endpoint["password"])
         self.conf.set(section_name, "tenant_name",
                       self.endpoint["tenant_name"])
-        self.conf.set(section_name, "alt_username", self.endpoint["username"])
+
-        self.conf.set(section_name, "alt_password", self.endpoint["password"])
-        self.conf.set(section_name, "alt_tenant_name",
-                      self.endpoint["tenant_name"])
         self.conf.set(section_name, "admin_username",
                       self.endpoint["username"])
         self.conf.set(section_name, "admin_password",
                       self.endpoint["password"])
         self.conf.set(section_name, "admin_tenant_name",
                       self.endpoint["tenant_name"])
+
         self.conf.set(section_name, "uri", self.endpoint["auth_url"])
+        v2_url_trailer = parse.urlparse(self.endpoint["auth_url"]).path
         self.conf.set(section_name, "uri_v3",
-                      self.endpoint["auth_url"].replace("/v2.0", "/v3"))
+                      self.endpoint["auth_url"].replace(v2_url_trailer, "/v3"))
+
         self.conf.set(section_name, "admin_domain_name",
                       self.endpoint["admin_domain_name"])
 
-    def _set_network(self, section_name="network"):
+    # The compute section is configured in context class for Tempest resources.
-        if "neutron" in self.available_services:
+    # Options which are configured there: 'image_ref', 'image_ref_alt',
-            neutron = self.clients.neutron()
+    # 'flavor_ref', 'flavor_ref_alt'.
-            public_net = [net for net in neutron.list_networks()["networks"] if
-                          net["status"] == "ACTIVE" and
-                          net["router:external"] is True]
-            if public_net:
-                net_id = public_net[0]["id"]
-                self.conf.set(section_name, "public_network_id", net_id)
-                public_router = neutron.list_routers(
-                    network_id=net_id)["routers"][0]
-                self.conf.set(section_name, "public_router_id",
-                              public_router["id"])
-                subnets = neutron.list_subnets(network_id=net_id)["subnets"]
-                if subnets:
-                    subnet = subnets[0]
-                else:
-                    # TODO(akurilin): create public subnet
-                    LOG.warn("No public subnet is found.")
-            else:
-                subnets = neutron.list_subnets()["subnets"]
-                if subnets:
-                    subnet = subnets[0]
-                else:
-                    # TODO(akurilin): create subnet
-                    LOG.warn("No subnet is found.")
-            self.conf.set(section_name, "tenant_network_cidr", subnet["cidr"])
-        else:
-            network = self.clients.nova().networks.list()[0]
-            self.conf.set(section_name, "tenant_network_cidr", network.cidr)
 
-    def _set_service_available(self, section_name="service_available"):
+    def _configure_network(self, section_name="network"):
-        services = ["neutron", "heat", "ceilometer", "swift",
+        if "neutron" in self.available_services:
-                    "cinder", "nova", "glance"]
+            neutronclient = self.clients.neutron()
+            public_nets = [net for net
+                           in neutronclient.list_networks()["networks"]
+                           if net["status"] == "ACTIVE" and
+                           net["router:external"] is True]
+            if public_nets:
+                net_id = public_nets[0]["id"]
+                self.conf.set(section_name, "public_network_id", net_id)
+        else:
+            novaclient = self.clients.nova()
+            net_name = next(net.human_id for net in novaclient.networks.list()
+                            if net.human_id is not None)
+            self.conf.set("compute", "fixed_network_name", net_name)
+            self.conf.set("compute", "network_for_ssh", net_name)
+
+    def _configure_network_feature_enabled(
+            self, section_name="network-feature-enabled"):
+        if "neutron" in self.available_services:
+            neutronclient = self.clients.neutron()
+            ext_list = [ext["alias"] for ext in
+                        neutronclient.list_ext("/extensions")["extensions"]]
+            ext_list_str = ",".join(ext_list)
+            self.conf.set(section_name, "api_extensions", ext_list_str)
+
+    def _configure_oslo_concurrency(self, section_name="oslo_concurrency"):
+        lock_path = os.path.join(self.data_dir,
+                                 "lock_files_%s" % self.deployment)
+        if not os.path.exists(lock_path):
+            os.makedirs(lock_path)
+        self.conf.set(section_name, "lock_path", lock_path)
+
+    def _configure_object_storage(self, section_name="object-storage"):
+        self.conf.set(section_name, "operator_role",
+                      CONF.role.swift_operator_role)
+        self.conf.set(section_name, "reseller_admin_role",
+                      CONF.role.swift_reseller_admin_role)
+
+    def _configure_scenario(self, section_name="scenario"):
+        self.conf.set(section_name, "img_dir", self.data_dir)
+        self.conf.set(section_name, "img_file", IMAGE_NAME)
+
+    def _configure_service_available(self, section_name="service_available"):
+        services = ["ceilometer", "cinder", "glance",
+                    "heat", "neutron", "nova", "sahara", "swift"]
         for service in services:
+            # Convert boolean to string because ConfigParser fails
+            # on attempt to get option with boolean value
             self.conf.set(section_name, service,
                           str(service in self.available_services))
         horizon_url = ("http://" +
                        parse.urlparse(self.endpoint["auth_url"]).hostname)
         try:
             horizon_req = requests.get(horizon_url)
-        except requests.RequestException as e:
+        except requests.RequestException:
-            LOG.debug("Failed to connect to Horizon: %s" % e)
+            LOG.debug("Horizon is unavailable!")
             horizon_availability = False
         else:
             horizon_availability = (horizon_req.status_code == 200)
-        # convert boolean to string because ConfigParser fails
+        # Convert boolean to string because ConfigParser fails
         # on attempt to get option with boolean value
         self.conf.set(section_name, "horizon", str(horizon_availability))
 
-    def write_config(self, file_name):
+    def _configure_validation(self, section_name="validation"):
-        with open(file_name, "w+") as f:
+        if "neutron" in self.available_services:
-            self.conf.write(f)
+            self.conf.set(section_name, "connect_method", "floating")
+        else:
+            self.conf.set(section_name, "connect_method", "fixed")
 
-    def generate(self, file_name=None):
+    def generate(self, conf_path=None):
-        for name, func in inspect.getmembers(self, predicate=inspect.ismethod):
+        for name, method in inspect.getmembers(self, inspect.ismethod):
-            if name.startswith("_set_"):
+            if name.startswith("_configure_"):
-                func()
+                method()
-        if file_name:
-            self.write_config(file_name)
 
-        return self.conf
+        if conf_path:
+            _write_config(conf_path, self.conf)
+
+
+class TempestResourcesContext(object):
+    """Context class to create/delete resources needed for Tempest."""
+
+    def __init__(self, deployment, conf_path):
+        endpoint = db.deployment_get(deployment)["admin"]
+        self.clients = osclients.Clients(objects.Endpoint(**endpoint))
+        self.keystone = self.clients.verified_keystone()
+
+        self.conf_path = conf_path
+        self.conf = configparser.ConfigParser()
+        self.conf.read(conf_path)
+
+    def __enter__(self):
+        self._created_roles = []
+        self._created_images = []
+        self._created_flavors = []
+
+        self._create_tempest_roles()
+        self._configure_option("image_ref", self._create_image)
+        self._configure_option("image_ref_alt", self._create_image)
+        self._configure_option("flavor_ref", self._create_flavor, 64)
+        self._configure_option("flavor_ref_alt", self._create_flavor, 128)
+
+        _write_config(self.conf_path, self.conf)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self._cleanup_roles()
+        self._cleanup_resource("image", self._created_images)
+        self._cleanup_resource("flavor", self._created_flavors)
+
+    def _create_tempest_roles(self):
+        roles = [CONF.role.swift_operator_role,
+                 CONF.role.swift_reseller_admin_role,
+                 CONF.role.heat_stack_owner_role,
+                 CONF.role.heat_stack_user_role]
+        existing_roles = set(role.name for role in self.keystone.roles.list())
+
+        for role in roles:
+            if role not in existing_roles:
+                LOG.debug("Creating role '%s'" % role)
+                self._created_roles.append(self.keystone.roles.create(role))
+
+    def _configure_option(self, option, create_method, *args, **kwargs):
+        option_value = self.conf.get("compute", option)
+        if not option_value:
+            LOG.debug("Option '%s' is not configured" % option)
+            resource = create_method(*args, **kwargs)
+            self.conf.set("compute", option, resource.id)
+            LOG.debug("Option '{opt}' is configured. {opt} = {resource_id}"
+                      .format(opt=option, resource_id=resource.id))
+        else:
+            LOG.debug("Option '{opt}' was configured manually "
+                      "in Tempest config file. {opt} = {opt_val}"
+                      .format(opt=option, opt_val=option_value))
+
+    def _create_image(self):
+        glanceclient = self.clients.glance()
+        image_name = "rally-verify-cirros-img-%s" % uuid.uuid4()
+        LOG.debug("Creating image '%s'" % image_name)
+        try:
+            image = glanceclient.images.create(
+                name=image_name,
+                disk_format="qcow2",
+                container_format="bare",
+                is_public=True)
+            self._created_images.append(image)
+            image.update(data=open(
+                os.path.join(_create_or_get_data_dir(), IMAGE_NAME), "rb"))
+        except Exception as exc:
+            msg = _("Image could not be created. "
+                    "Reason: %s") % (str(exc) or "unknown")
+            raise exceptions.TempestResourceCreationFailure(msg)
+
+        return image
+
+    def _create_flavor(self, flv_ram):
+        novaclient = self.clients.nova()
+        flavor_name = "m1.rally-verify-flv-%s" % uuid.uuid4()
+        LOG.debug("Creating flavor '%s'" % flavor_name)
+        try:
+            flavor = novaclient.flavors.create(
+                flavor_name, ram=flv_ram, vcpus=1, disk=0)
+        except Exception as exc:
+            msg = _("Flavor could not be created. "
+                    "Reason: %s") % (str(exc) or "unknown")
+            raise exceptions.TempestResourceCreationFailure(msg)
+
+        self._created_flavors.append(flavor)
+
+        return flavor
+
+    def _cleanup_roles(self):
+        for role in self._created_roles:
+            LOG.debug("Deleting role '%s'" % role.name)
+            role.delete()
+
+    def _cleanup_resource(self, resource_type, created_resources):
+        for res in created_resources:
+            LOG.debug("Deleting %s '%s'" % (resource_type, res.name))
+            if res.id == self.conf.get("compute", "%s_ref" % resource_type):
+                self.conf.set("compute", "%s_ref" % resource_type, "")
+            else:
+                self.conf.set("compute", "%s_ref_alt" % resource_type, "")
+            res.delete()
+
+        _write_config(self.conf_path, self.conf)

rally/verification/tempest/tempest.py

@@ -229,7 +229,7 @@ class Tempest(object):
             msg = _("Creation of configuration file for tempest.")
             LOG.info(_("Starting: ") + msg)
 
-            config.TempestConf(self.deployment).generate(self.config_file)
+            config.TempestConfig(self.deployment).generate(self.config_file)
             LOG.info(_("Completed: ") + msg)
         else:
             LOG.info("Tempest is already configured.")
@@ -345,6 +345,10 @@ class Tempest(object):
                 "log_file": log_file or self.log_file_raw
             })
         LOG.debug("Test(s) started by the command: %s" % test_cmd)
+        # Create all resources needed for Tempest before running tests.
+        # Once tests finish, all created resources will be deleted.
+        with config.TempestResourcesContext(self.deployment, self.config_file):
+            # Run tests
             subprocess.check_call(test_cmd, cwd=self.path(),
                                   env=self.env, shell=True)
 

tests/ci/rally-gate/index_verify.mako

@@ -62,27 +62,26 @@
         </li>
         <li>
             <span class="${vr_1_html}">[${vr_1_html}]</span>
-            <a href="rally-verify/1_verify_results.html.gz">Display raw results in HTML</a>
+            <a href="rally-verify/1_verify_results_compute_set.html.gz">Display raw results in HTML</a>
             <code>$ rally verify results --html</code>
         </li>
         <li>
             <span class="${vr_1_json}">[${vr_1_json}]</span>
-            <a href="rally-verify/1_verify_results.json.gz">Display raw results in JSON</a>
+            <a href="rally-verify/1_verify_results_compute_set.json.gz">Display raw results in JSON</a>
             <code>$ rally verify results --json</code>
         </li>
         <li>
             <span class="${vs_1}">[${vs_1}]</span>
-            <a href="rally-verify/1_verify_show.txt.gz">Display results table of the verification</a>
+            <a href="rally-verify/1_verify_show_compute_set.txt.gz">Display results table of the verification</a>
             <code>$ rally verify show</code>
         </li>
         <li>
             <span class="${vsd_1}">[${vsd_1}]</span>
-            <a href="rally-verify/1_verify_show_detailed.txt.gz">Display results table of the verification with detailed errors</a><br />
+            <a href="rally-verify/1_verify_show_compute_set_detailed.txt.gz">Display results table of the verification with detailed errors</a><br />
             <code style="display: inline">$ rally verify show --detailed</code> or <code style="display: inline">$ rally verify detailed</code>
         </li>
     </ol>
 
-
     Second verification run
     <ol>
         <li>
@@ -92,22 +91,22 @@
         </li>
         <li>
             <span class="${vr_2_html}">[${vr_2_html}]</span>
-            <a href="rally-verify/2_verify_results.html.gz">Display results in HTML</a>
+            <a href="rally-verify/2_verify_results_compute_set.html.gz">Display results in HTML</a>
             <code>$ rally verify results --html</code>
         </li>
         <li>
             <span class="${vr_2_json}">[${vr_2_json}]</span>
-            <a href="rally-verify/2_verify_results.json.gz">Display results in JSON</a>
+            <a href="rally-verify/2_verify_results_compute_set.json.gz">Display results in JSON</a>
             <code>$ rally verify results --json</code>
         </li>
         <li>
             <span class="${vs_2}">[${vs_2}]</span>
-            <a href="rally-verify/2_verify_show.txt.gz">Display table results of the verification</a>
+            <a href="rally-verify/2_verify_show_compute_set.txt.gz">Display table results of the verification</a>
             <code>$ rally verify show</code>
         </li>
         <li>
             <span class="${vsd_2}">[${vsd_2}]</span>
-            <a href="rally-verify/2_verify_show_detailed.txt.gz">Display table results of the verification with detailed errors</a><br />
+            <a href="rally-verify/2_verify_show_compute_set_detailed.txt.gz">Display table results of the verification with detailed errors</a><br />
             <code style="display: inline">$ rally verify show --detailed</code> or <code style="display: inline">$ rally verify detailed</code>
         </li>
     </ol>

tests/ci/rally-verify.sh

@@ -48,8 +48,8 @@ gzip -9 ${RESULTS_DIR}/tempest_installation.txt
 gzip -9 ${RESULTS_DIR}/tempest_config_generation.txt
 
 function do_verification {
-    OUTPUT_FILE=${RESULTS_DIR}/${1}_verification_${SET_NAME}_set.txt
+    OUTPUT_FILE=${RESULTS_DIR}/${1}_verification_${2}_set.txt
-    rally --rally-debug verify start --regex tempest.api.compute.servers.test_servers > ${OUTPUT_FILE} 2>&1
+    rally --rally-debug verify start --set ${2} > ${OUTPUT_FILE} 2>&1
     RESULTS+="v${1}=$(do_status $?) "
     gzip -9 ${OUTPUT_FILE}
     source ~/.rally/globals && VERIFICATIONS[${1}]=${RALLY_VERIFICATION}
@@ -57,24 +57,24 @@ function do_verification {
     # Check different "rally verify" commands, which displays verification results
     for OUTPUT_FORMAT in "html" "json"
     do
-        OUTPUT_FILE=${RESULTS_DIR}/${1}_verify_results.${OUTPUT_FORMAT}
+        OUTPUT_FILE=${RESULTS_DIR}/${1}_verify_results_${2}_set.${OUTPUT_FORMAT}
         rally verify results --uuid ${RALLY_VERIFICATION} --${OUTPUT_FORMAT} --output-file ${OUTPUT_FILE}
         RESULTS+="vr_${1}_${OUTPUT_FORMAT}=$(do_status $?) "
         gzip -9 ${OUTPUT_FILE}
     done
 
-    rally verify show --uuid ${RALLY_VERIFICATION} > ${RESULTS_DIR}/${1}_verify_show.txt
+    rally verify show --uuid ${RALLY_VERIFICATION} > ${RESULTS_DIR}/${1}_verify_show_${2}_set.txt
     RESULTS+="vs_${1}=$(do_status $?) "
-    gzip -9 ${RESULTS_DIR}/${1}_verify_show.txt
+    gzip -9 ${RESULTS_DIR}/${1}_verify_show_${2}_set.txt
 
-    rally verify show --uuid ${RALLY_VERIFICATION} --detailed > ${RESULTS_DIR}/${1}_verify_show_detailed.txt
+    rally verify show --uuid ${RALLY_VERIFICATION} --detailed > ${RESULTS_DIR}/${1}_verify_show_${2}_set_detailed.txt
     RESULTS+="vsd_${1}=$(do_status $?) "
-    gzip -9 ${RESULTS_DIR}/${1}_verify_show_detailed.txt
+    gzip -9 ${RESULTS_DIR}/${1}_verify_show_${2}_set_detailed.txt
 }
 
 function main {
-    do_verification 1
+    do_verification 1 compute
-    do_verification 2
+    do_verification 2 compute
 
     rally verify list > ${RESULTS_DIR}/verify_list.txt
     RESULTS+="l=$(do_status $?) "

tests/unit/plugins/openstack/context/not_for_production/test_tempest.py

@@ -17,7 +17,6 @@ import mock
 
 from rally import exceptions
 from rally.plugins.openstack.context.not_for_production import tempest
-from rally.verification.tempest import config
 from rally.verification.tempest import tempest as tempest_verifier
 from tests.unit import test
 
@@ -75,7 +74,7 @@ class TempestContextTestCase(test.TestCase):
             self, mock_tempest_generate_config_file, mock_tempest_is_installed,
             mock_tempest_is_configured, mock_mkdir):
         mock_tempest_generate_config_file.side_effect = (
-            config.TempestConfigCreationFailure()
+            exceptions.TempestConfigCreationFailure()
         )
 
         benchmark = tempest.Tempest(self.context)

tests/unit/plugins/openstack/scenarios/tempest/test_tempest.py

@@ -19,7 +19,8 @@ from rally.plugins.openstack.scenarios.tempest import tempest
 from rally.verification.tempest import tempest as verifier
 from tests.unit import test
 
-VERIFIER = "rally.verification.tempest.tempest"
+TEMPEST_DIR = "rally.verification.tempest"
+VERIFIER = TEMPEST_DIR + ".tempest"
 TS = "rally.plugins.openstack.scenarios.tempest"
 
 
@@ -51,7 +52,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_single_test(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_single_test(self, mock_tempest_resources_context,
+                         mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
         fake_test = "tempest.api.fake.test"
 
@@ -64,7 +67,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_single_test_negative(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_single_test_negative(self, mock_tempest_resources_context,
+                                  mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
         fake_test = "tempest.api.network"
 
@@ -77,7 +82,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_single_test_without_prefix(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_single_test_without_prefix(self, mock_tempest_resources_context,
+                                        mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         self.scenario.single_test("network")
@@ -89,7 +96,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_all(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_all(self, mock_tempest_resources_context,
+                 mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         self.scenario.all()
@@ -101,7 +110,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_set_smoke(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_set_smoke(self, mock_tempest_resources_context,
+                       mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         self.scenario.set("smoke")
@@ -113,7 +124,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_set_full(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_set_full(self, mock_tempest_resources_context,
+                      mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         self.scenario.set("full")
@@ -124,7 +137,9 @@ class TempestScenarioTestCase(test.TestCase):
             env=self.verifier.env, shell=True)
 
     @mock.patch(TS + ".utils.tempfile")
-    def test_set_from_list(self, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_set_from_list(self, mock_tempest_resources_context,
+                           mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         fake_scenarios = ["network", "volume", "baremetal",
@@ -143,7 +158,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_set_selective(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_set_selective(self, mock_tempest_resources_context,
+                           mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
 
         self.scenario.set("network")
@@ -155,7 +172,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_list_of_tests(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_list_of_tests(self, mock_tempest_resources_context,
+                           mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
         fake_tests = ["tempest.fake.test1", "tempest.fake.test2"]
 
@@ -168,7 +187,9 @@ class TempestScenarioTestCase(test.TestCase):
 
     @mock.patch(TS + ".utils.tempfile")
     @mock.patch(VERIFIER + ".subprocess")
-    def test_specific_regex(self, mock_subprocess, mock_tempfile):
+    @mock.patch(TEMPEST_DIR + ".config.TempestResourcesContext")
+    def test_specific_regex(self, mock_tempest_resources_context,
+                            mock_subprocess, mock_tempfile):
         mock_tempfile.NamedTemporaryFile().name = "/dev/null"
         regex = "tempest.fake.test1"
 

tests/unit/verification/test_config.py

@@ -19,6 +19,7 @@ import mock
 from oslo_config import cfg
 import requests
 
+from rally import exceptions
 from rally.verification.tempest import config
 from tests.unit import fakes
 from tests.unit import test
@@ -37,23 +38,31 @@ class ConfigTestCase(test.TestCase):
     def setUp(self, mock_isfile, mock_clients_verified_keystone,
               mock_clients_services, mock_deployment_get):
         super(ConfigTestCase, self).setUp()
-        self.endpoint = {"username": "test",
+
+        self.endpoint = {
+            "username": "test",
             "tenant_name": "test",
             "password": "test",
-                         "auth_url": "http://test/v2.0",
+            "auth_url": "http://test/v2.0/",
             "permission": "admin",
-                         "admin_domain_name": "Default"}
+            "admin_domain_name": "Default"
+        }
         mock_deployment_get.return_value = {"admin": self.endpoint}
+
         self.deployment = "fake_deployment"
-        self.conf_generator = config.TempestConf(self.deployment)
+        self.conf_generator = config.TempestConfig(self.deployment)
         self.conf_generator.clients.services = mock_clients_services
+        self.context = config.TempestResourcesContext(self.deployment,
+                                                      "/path/to/fake/conf")
+        self.context.conf.add_section("compute")
 
         keystone_patcher = mock.patch("rally.osclients.create_keystone_client")
         keystone_patcher.start()
         self.addCleanup(keystone_patcher.stop)
 
-    def _remove_default_section(self, items):
+    @staticmethod
-        # getting items from configparser by specified section name
+    def _remove_default_section(items):
+        # Getting items from config parser by specified section name
         # returns also values from DEFAULT section
         defaults = (("log_file", "tempest.log"), ("debug", "True"),
                     ("use_stderr", "False"))
@@ -63,269 +72,223 @@ class ConfigTestCase(test.TestCase):
     @mock.patch("rally.verification.tempest.config.os.rename")
     @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(),
                 create=True)
-    def test__load_img_success(self, mock_open, mock_rename, mock_requests):
+    def test__download_cirros_image_success(self, mock_open, mock_rename,
+                                            mock_requests):
         mock_result = mock.MagicMock()
         mock_result.status_code = 200
         mock_requests.get.return_value = mock_result
-        self.conf_generator._load_img()
+        self.conf_generator._download_cirros_image()
-        cirros_url = ("http://download.cirros-cloud.net/%s/%s" %
+        mock_requests.get.assert_called_once_with(CONF.image.cirros_img_url,
-                      (CONF.image.cirros_version,
+                                                  stream=True)
-                       CONF.image.cirros_image))
-        mock_requests.get.assert_called_once_with(cirros_url, stream=True)
 
     @mock.patch("rally.verification.tempest.config.requests")
-    def test__load_img_notfound(self, mock_requests):
+    def test__download_cirros_image_notfound(self, mock_requests):
         mock_result = mock.MagicMock()
         mock_result.status_code = 404
         mock_requests.get.return_value = mock_result
-        self.assertRaises(config.TempestConfigCreationFailure,
+        self.assertRaises(exceptions.TempestConfigCreationFailure,
-                          self.conf_generator._load_img)
+                          self.conf_generator._download_cirros_image)
 
-    def test__get_url(self):
+    def test__get_service_url(self):
         service = "test_service"
         service_type = "test_service_type"
         url = "test_url"
-        # mocked at setUp
+        # Mocked at setUp
-        self.conf_generator.keystoneclient.auth_ref = {
+        self.conf_generator.keystone.auth_ref = {
-            "serviceCatalog": [{
+            "serviceCatalog": [
+                {
                     "name": service,
                     "type": service_type,
                     "endpoints": [{"publicURL": url}]
-            }]}
+                }
-        self.assertEqual(self.conf_generator._get_url(service), url)
+            ]
+        }
+        self.assertEqual(self.conf_generator._get_service_url(service), url)
 
-    @mock.patch("rally.verification.tempest.config.TempestConf"
+    @mock.patch("rally.verification.tempest."
-                "._get_url")
+                "config.TempestConfig._get_service_url")
-    def test__set_boto(self, mock_tempest_conf__get_url):
+    def test__configure_boto(self, mock_tempest_config__get_service_url):
         url = "test_url"
-        mock_tempest_conf__get_url.return_value = url
+        mock_tempest_config__get_service_url.return_value = url
-        self.conf_generator._set_boto()
+        s3_materials_path = os.path.join(
+            self.conf_generator.data_dir, "s3materials")
+        self.conf_generator._configure_boto()
         expected = (("ec2_url", url),
                     ("s3_url", url),
-                    ("build_interval", "1"),
-                    ("build_timeout", "196"),
                     ("http_socket_timeout", "30"),
-                    ("instance_type", "m1.nano"),
+                    ("s3_materials_path", s3_materials_path))
-                    ("s3_materials_path",
-                     os.path.join(self.conf_generator.data_path,
-                                  "s3materials")))
         results = self._remove_default_section(
             self.conf_generator.conf.items("boto"))
         self.assertEqual(sorted(expected), sorted(results))
 
-    def test__set_compute_flavors(self):
+    def test__configure_default(self):
-        mock_novaclient = mock.MagicMock()
+        self.conf_generator._configure_default()
-        mock_novaclient.flavors.list.return_value = [
-            fakes.FakeFlavor(id="id1"), fakes.FakeFlavor(id="id2")]
-        mock_nova = mock.MagicMock()
-        mock_nova.client.Client.return_value = mock_novaclient
-        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
-            self.conf_generator._set_compute_flavors()
-            expected = ("id1", "id2")
-            results = (self.conf_generator.conf.get("compute", "flavor_ref"),
-                       self.conf_generator.conf.get("compute",
-                                                    "flavor_ref_alt"))
-            self.assertEqual(sorted(expected), sorted(results))
-
-    def test__set_compute_flavors_create(self):
-        mock_novaclient = mock.MagicMock()
-        mock_novaclient.flavors.list.return_value = []
-        mock_novaclient.flavors.create.side_effect = [
-            fakes.FakeFlavor(id="id1"), fakes.FakeFlavor(id="id2")]
-        mock_nova = mock.MagicMock()
-        mock_nova.client.Client.return_value = mock_novaclient
-        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
-            self.conf_generator._set_compute_flavors()
-            self.assertEqual(mock_novaclient.flavors.create.call_count, 2)
-            expected = ("id1", "id2")
-            results = (self.conf_generator.conf.get("compute", "flavor_ref"),
-                       self.conf_generator.conf.get("compute",
-                                                    "flavor_ref_alt"))
-            self.assertEqual(sorted(expected), sorted(results))
-
-    def test__set_compute_flavors_create_fails(self):
-        mock_novaclient = mock.MagicMock()
-        mock_novaclient.flavors.list.return_value = []
-        mock_novaclient.flavors.create.side_effect = Exception()
-        mock_nova = mock.MagicMock()
-        mock_nova.client.Client.return_value = mock_novaclient
-        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
-            self.assertRaises(config.TempestConfigCreationFailure,
-                              self.conf_generator._set_compute_flavors)
-
-    def test__set_compute_images(self):
-        mock_glanceclient = mock.MagicMock()
-        mock_glanceclient.images.list.return_value = [
-            fakes.FakeImage(id="id1", name="cirros1"),
-            fakes.FakeImage(id="id2", name="cirros2")]
-        mock_glance = mock.MagicMock()
-        mock_glance.Client.return_value = mock_glanceclient
-        with mock.patch.dict("sys.modules", {"glanceclient": mock_glance}):
-            self.conf_generator._set_compute_images()
-            expected = ("id1", "id2")
-            results = (self.conf_generator.conf.get("compute", "image_ref"),
-                       self.conf_generator.conf.get("compute",
-                                                    "image_ref_alt"))
-            self.assertEqual(sorted(expected), sorted(results))
-
-    @mock.patch("six.moves.builtins.open")
-    def test__set_compute_images_create(self, mock_open):
-        mock_glanceclient = mock.MagicMock()
-        mock_glanceclient.images.list.return_value = []
-        mock_glanceclient.images.create.side_effect = [
-            fakes.FakeImage(id="id1"), fakes.FakeImage(id="id2")]
-        mock_glance = mock.MagicMock()
-        mock_glance.Client.return_value = mock_glanceclient
-        with mock.patch.dict("sys.modules", {"glanceclient": mock_glance}):
-            self.conf_generator._set_compute_images()
-            self.assertEqual(mock_glanceclient.images.create.call_count, 2)
-            expected = ("id1", "id2")
-            results = (self.conf_generator.conf.get("compute", "image_ref"),
-                       self.conf_generator.conf.get("compute",
-                                                    "image_ref_alt"))
-            self.assertEqual(sorted(expected), sorted(results))
-
-    def test__set_compute_images_create_fails(self):
-        mock_glanceclient = mock.MagicMock()
-        mock_glanceclient.images.list.return_value = []
-        mock_glanceclient.images.create.side_effect = Exception()
-        mock_glance = mock.MagicMock()
-        mock_glance.Client.return_value = mock_glanceclient
-        with mock.patch.dict("sys.modules", {"glanceclient": mock_glance}):
-            self.assertRaises(config.TempestConfigCreationFailure,
-                              self.conf_generator._set_compute_images)
-
-    def test__set_compute_ssh_connect_method_if_neutron(self):
-        self.conf_generator._set_compute_ssh_connect_method()
-        self.assertEqual("fixed",
-                         self.conf_generator.conf.get("compute",
-                                                      "ssh_connect_method"))
-        # if neutron is available
-        self.conf_generator.available_services = ["neutron"]
-        self.conf_generator._set_compute_ssh_connect_method()
-        self.assertEqual("floating",
-                         self.conf_generator.conf.get("compute",
-                                                      "ssh_connect_method"))
-
-    def test__set_default(self):
-        self.conf_generator._set_default()
         expected = (("debug", "True"), ("log_file", "tempest.log"),
                     ("use_stderr", "False"))
         results = self.conf_generator.conf.items("DEFAULT")
         self.assertEqual(sorted(expected), sorted(results))
 
+    def test__configure_identity(self):
+        self.conf_generator._configure_identity()
+        expected = (
+            ("username", self.endpoint["username"]),
+            ("password", self.endpoint["password"]),
+            ("tenant_name", self.endpoint["tenant_name"]),
+            ("admin_username", self.endpoint["username"]),
+            ("admin_password", self.endpoint["password"]),
+            ("admin_tenant_name", self.endpoint["username"]),
+            ("admin_domain_name", self.endpoint["admin_domain_name"]),
+            ("uri", self.endpoint["auth_url"]),
+            ("uri_v3", self.endpoint["auth_url"].replace("/v2.0/", "/v3")))
+        results = self._remove_default_section(
+            self.conf_generator.conf.items("identity"))
+        self.assertEqual(sorted(expected), sorted(results))
+
+    def test__configure_option_flavor(self):
+        mock_novaclient = mock.MagicMock()
+        mock_novaclient.flavors.create.side_effect = [
+            fakes.FakeFlavor(id="id1"), fakes.FakeFlavor(id="id2")]
+        mock_nova = mock.MagicMock()
+        mock_nova.client.Client.return_value = mock_novaclient
+        self.context.conf.set("compute", "flavor_ref", "")
+        self.context.conf.set("compute", "flavor_ref_alt", "")
+        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
+            self.context._configure_option("flavor_ref",
+                                           mock_novaclient.flavors.create, 64)
+            self.context._configure_option("flavor_ref_alt",
+                                           mock_novaclient.flavors.create, 128)
+            self.assertEqual(mock_novaclient.flavors.create.call_count, 2)
+            expected = ("id1", "id2")
+            results = (self.context.conf.get("compute", "flavor_ref"),
+                       self.context.conf.get("compute", "flavor_ref_alt"))
+            self.assertEqual(sorted(expected), sorted(results))
+
+    @mock.patch("six.moves.builtins.open")
+    def test__configure_option_image(self, mock_open):
+        mock_glanceclient = mock.MagicMock()
+        mock_glanceclient.images.create.side_effect = [
+            fakes.FakeImage(id="id1"), fakes.FakeImage(id="id2")]
+        mock_glance = mock.MagicMock()
+        mock_glance.Client.return_value = mock_glanceclient
+        self.context.conf.set("compute", "image_ref", "")
+        self.context.conf.set("compute", "image_ref_alt", "")
+        with mock.patch.dict("sys.modules", {"glanceclient": mock_glance}):
+            self.context._configure_option("image_ref",
+                                           mock_glanceclient.images.create)
+            self.context._configure_option("image_ref_alt",
+                                           mock_glanceclient.images.create)
+            self.assertEqual(mock_glanceclient.images.create.call_count, 2)
+            expected = ("id1", "id2")
+            results = (self.context.conf.get("compute", "image_ref"),
+                       self.context.conf.get("compute", "image_ref_alt"))
+            self.assertEqual(sorted(expected), sorted(results))
+
+    def test__configure_network_if_neutron(self):
+        fake_neutronclient = mock.MagicMock()
+        fake_neutronclient.list_networks.return_value = {
+            "networks": [
+                {
+                    "status": "ACTIVE",
+                    "id": "test_id",
+                    "router:external": True
+                }
+            ]
+        }
+        mock_neutron = mock.MagicMock()
+        mock_neutron.client.Client.return_value = fake_neutronclient
+        with mock.patch.dict("sys.modules", {"neutronclient.neutron":
+                                             mock_neutron}):
+            self.conf_generator.available_services = ["neutron"]
+            self.conf_generator._configure_network()
+            expected = (("public_network_id", "test_id"),)
+            results = self._remove_default_section(
+                self.conf_generator.conf.items("network"))
+            self.assertEqual(sorted(expected), sorted(results))
+
+    def test__configure_network_if_nova(self):
+        self.conf_generator.available_services = ["nova"]
+        mock_novaclient = mock.MagicMock()
+        mock_network = mock.MagicMock()
+        mock_network.human_id = "fake-network"
+        mock_novaclient.networks.list.return_value = [mock_network]
+        mock_nova = mock.MagicMock()
+        mock_nova.client.Client.return_value = mock_novaclient
+        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
+            self.conf_generator._configure_network()
+            self.assertEqual("fake-network",
+                             self.conf_generator.conf.get(
+                                 "compute", "fixed_network_name"))
+            self.assertEqual("fake-network",
+                             self.conf_generator.conf.get(
+                                 "compute", "network_for_ssh"))
+
     @mock.patch("rally.verification.tempest.config.os.path.exists",
                 return_value=False)
     @mock.patch("rally.verification.tempest.config.os.makedirs")
-    def test__set_oslo_concurrency(self, mock_makedirs, mock_exists):
+    def test__configure_oslo_concurrency(self, mock_makedirs, mock_exists):
-        self.conf_generator._set_oslo_concurrency()
+        self.conf_generator._configure_oslo_concurrency()
-        lock_path = os.path.join(self.conf_generator.data_path, "lock_files_%s"
+        lock_path = os.path.join(
-                                 % self.deployment)
+            self.conf_generator.data_dir, "lock_files_%s" % self.deployment)
         mock_makedirs.assert_called_once_with(lock_path)
         expected = (("lock_path", lock_path),)
         results = self._remove_default_section(
             self.conf_generator.conf.items("oslo_concurrency"))
         self.assertEqual(sorted(expected), sorted(results))
 
-    def test__set_identity(self):
-        self.conf_generator._set_identity()
-        expected = (("username", self.endpoint["username"]),
-                    ("password", self.endpoint["password"]),
-                    ("tenant_name", self.endpoint["tenant_name"]),
-                    ("alt_username", self.endpoint["username"]),
-                    ("alt_password", self.endpoint["password"]),
-                    ("alt_tenant_name", self.endpoint["tenant_name"]),
-                    ("admin_username", self.endpoint["username"]),
-                    ("admin_password", self.endpoint["password"]),
-                    ("admin_tenant_name", self.endpoint["username"]),
-                    ("admin_domain_name", self.endpoint["admin_domain_name"]),
-                    ("uri", self.endpoint["auth_url"]),
-                    ("uri_v3", self.endpoint["auth_url"].replace("/v2.0",
-                                                                 "/v3")))
-        results = self._remove_default_section(
-            self.conf_generator.conf.items("identity"))
-        self.assertEqual(sorted(expected), sorted(results))
-
-    def test__set_network_if_neutron(self):
-        fake_neutronclient = mock.MagicMock()
-        fake_neutronclient.list_networks.return_value = {"networks": [
-                                                         {"status": "ACTIVE",
-                                                          "id": "test_id",
-                                                          "router:external":
-                                                          True}]}
-        fake_neutronclient.list_routers.return_value = {"routers": [
-                                                        {"id": "test_router"}]}
-        fake_neutronclient.list_subnets.return_value = {"subnets": [
-                                                        {"cidr":
-                                                         "10.0.0.0/24"}]}
-        mock_neutron = mock.MagicMock()
-        mock_neutron.client.Client.return_value = fake_neutronclient
-        with mock.patch.dict("sys.modules", {"neutronclient.neutron":
-                                             mock_neutron}):
-            self.conf_generator.available_services = ["neutron"]
-            self.conf_generator._set_network()
-            expected = (("tenant_network_cidr", "10.0.0.0/24"),
-                        ("tenant_networks_reachable", "False"),
-                        ("public_network_id", "test_id"),
-                        ("public_router_id", "test_router"))
-            results = self._remove_default_section(
-                self.conf_generator.conf.items("network"))
-            self.assertEqual(sorted(expected), sorted(results))
-
-    def test__set_network_if_nova(self):
-        network = "10.0.0.0/24"
-        mock_novaclient = mock.MagicMock()
-        mock_network = mock.MagicMock()
-        mock_network.cidr = network
-        mock_novaclient.networks.list.return_value = [mock_network]
-        mock_nova = mock.MagicMock()
-        mock_nova.client.Client.return_value = mock_novaclient
-        with mock.patch.dict("sys.modules", {"novaclient": mock_nova}):
-            self.conf_generator._set_network()
-            self.assertEqual(network,
-                             self.conf_generator.conf.get(
-                                 "network", "tenant_network_cidr"))
-
     @mock.patch("rally.verification.tempest.config.requests")
-    def test__set_service_available(self, mock_requests):
+    def test__configure_service_available(self, mock_requests):
         mock_result = mock.MagicMock()
         mock_result.status_code = 404
         mock_requests.get.return_value = mock_result
-        available_services = ("nova", "cinder", "glance")
+        available_services = ("nova", "cinder", "glance", "sahara")
         self.conf_generator.available_services = available_services
-        self.conf_generator._set_service_available()
+        self.conf_generator._configure_service_available()
         expected = (("neutron", "False"), ("heat", "False"),
                     ("ceilometer", "False"), ("swift", "False"),
                     ("cinder", "True"), ("nova", "True"),
-                    ("glance", "True"), ("horizon", "False"))
+                    ("glance", "True"), ("horizon", "False"),
+                    ("sahara", "True"))
         options = self._remove_default_section(
             self.conf_generator.conf.items("service_available"))
         self.assertEqual(sorted(expected), sorted(options))
 
     @mock.patch("rally.verification.tempest.config.requests")
-    def test__set_service_available_horizon(self, mock_requests):
+    def test__configure_service_available_horizon(self, mock_requests):
         mock_result = mock.MagicMock()
         mock_result.status_code = 200
         mock_requests.get.return_value = mock_result
-        self.conf_generator._set_service_available()
+        self.conf_generator._configure_service_available()
-        self.assertEqual(self.conf_generator.conf.get(
+        self.assertEqual(
+            self.conf_generator.conf.get(
                 "service_available", "horizon"), "True")
 
     @mock.patch("rally.verification.tempest.config.requests.get")
-    def test__set_service_not_available_horizon(self, mock_get):
+    def test__configure_service_not_available_horizon(self, mock_get):
         mock_get.side_effect = requests.Timeout()
-        self.conf_generator._set_service_available()
+        self.conf_generator._configure_service_available()
-        self.assertEqual(self.conf_generator.conf.get(
+        self.assertEqual(
+            self.conf_generator.conf.get(
                 "service_available", "horizon"), "False")
 
-    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(),
+    def test__configure_validation_if_neutron(self):
-                create=True)
+        # if neutron is available
-    def test_write_config(self, mock_open):
+        self.conf_generator.available_services = ["neutron"]
-        self.conf_generator.conf = mock.Mock()
+        self.conf_generator._configure_validation()
-        file_name = "/path/to/fake/conf"
+        self.assertEqual("floating",
+                         self.conf_generator.conf.get("validation",
+                                                      "connect_method"))
 
-        self.conf_generator.write_config(file_name)
+    def test__configure_validation_if_novanetwork(self):
+        self.conf_generator._configure_validation()
+        self.assertEqual("fixed",
+                         self.conf_generator.conf.get("validation",
+                                                      "connect_method"))
 
-        mock_open.assert_called_once_with(file_name, "w+")
+    @mock.patch("six.moves.builtins.open",
-        self.conf_generator.conf.write.assert_called_once_with(
+                side_effect=mock.mock_open(), create=True)
-            mock_open.side_effect())
+    def test__write_config(self, mock_open):
+        conf_path = "/path/to/fake/conf"
+        conf_data = mock.Mock()
+        config._write_config(conf_path, conf_data)
+        mock_open.assert_called_once_with(conf_path, "w+")
+        conf_data.write.assert_called_once_with(mock_open.side_effect())

tests/unit/verification/test_tempest.py

@@ -339,12 +339,14 @@ class TempestVerifyTestCase(BaseTestCase):
                 return_value=(None, None))
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
     @mock.patch(TEMPEST_PATH + ".tempest.subprocess")
-    @mock.patch(TEMPEST_PATH + ".config.TempestConf")
+    @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext")
+    @mock.patch(TEMPEST_PATH + ".config.TempestConfig")
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
                 return_value=False)
     def test_verify_not_configured(
-            self, mock_tempest_is_configured, mock_tempest_conf,
+            self, mock_tempest_is_configured, mock_tempest_config,
-            mock_subprocess, mock_tempest_env, mock_tempest_parse_results):
+            mock_tempest_resources_context, mock_subprocess, mock_tempest_env,
+            mock_tempest_parse_results):
 
         set_name = "compute"
         fake_call = self._get_fake_call(set_name)
@@ -352,8 +354,8 @@ class TempestVerifyTestCase(BaseTestCase):
         self.verifier.verify(set_name, None)
 
         self.assertEqual(2, mock_tempest_is_configured.call_count)
-        mock_tempest_conf.assert_called_once_with(self.verifier.deployment)
+        mock_tempest_config.assert_called_once_with(self.verifier.deployment)
-        mock_tempest_conf.return_value.generate.assert_called_once_with(
+        mock_tempest_config.return_value.generate.assert_called_once_with(
             self.verifier.config_file
         )
         self.verifier.verification.start_verifying.assert_called_once_with(
@@ -368,20 +370,22 @@ class TempestVerifyTestCase(BaseTestCase):
                 return_value=(None, None))
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
     @mock.patch(TEMPEST_PATH + ".tempest.subprocess")
-    @mock.patch(TEMPEST_PATH + ".config.TempestConf")
+    @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext")
+    @mock.patch(TEMPEST_PATH + ".config.TempestConfig")
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
                 return_value=True)
     def test_verify_when_tempest_configured(
-            self, mock_tempest_is_configured, mock_tempest_conf,
+            self, mock_tempest_is_configured, mock_tempest_config,
-            mock_subprocess, mock_tempest_env, mock_tempest_parse_results):
+            mock_tempest_resources_context, mock_subprocess, mock_tempest_env,
+            mock_tempest_parse_results):
         set_name = "identity"
         fake_call = self._get_fake_call(set_name)
 
         self.verifier.verify(set_name, None)
 
         mock_tempest_is_configured.assert_called_once_with()
-        self.assertFalse(mock_tempest_conf.called)
+        self.assertFalse(mock_tempest_config.called)
-        self.assertFalse(mock_tempest_conf().generate.called)
+        self.assertFalse(mock_tempest_config().generate.called)
         self.verifier.verification.start_verifying.assert_called_once_with(
             set_name)
 
@@ -394,12 +398,14 @@ class TempestVerifyTestCase(BaseTestCase):
                 return_value=(None, None))
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
     @mock.patch(TEMPEST_PATH + ".tempest.subprocess")
-    @mock.patch(TEMPEST_PATH + ".config.TempestConf")
+    @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext")
+    @mock.patch(TEMPEST_PATH + ".config.TempestConfig")
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
                 return_value=True)
     def test_verify_failed_and_tempest_is_configured(
-            self, mock_tempest_is_configured, mock_tempest_conf,
+            self, mock_tempest_is_configured, mock_tempest_config,
-            mock_subprocess, mock_tempest_env, mock_tempest_parse_results):
+            mock_tempest_resources_context, mock_subprocess, mock_tempest_env,
+            mock_tempest_parse_results):
         set_name = "identity"
         fake_call = self._get_fake_call(set_name)
         mock_subprocess.side_effect = subprocess.CalledProcessError
@@ -407,8 +413,8 @@ class TempestVerifyTestCase(BaseTestCase):
         self.verifier.verify(set_name, None)
 
         mock_tempest_is_configured.assert_called_once_with()
-        self.assertFalse(mock_tempest_conf.called)
+        self.assertFalse(mock_tempest_config.called)
-        self.assertFalse(mock_tempest_conf().generate.called)
+        self.assertFalse(mock_tempest_config().generate.called)
         self.verifier.verification.start_verifying.assert_called_once_with(
             set_name)