diff --git a/rally/consts.py b/rally/consts.py index f55353dc..494fc9a9 100644 --- a/rally/consts.py +++ b/rally/consts.py @@ -24,27 +24,6 @@ for each enum. (e.g TaskStatus) from rally.common import utils -class _TempestTestsAPI(utils.ImmutableMixin, utils.EnumMixin): - BAREMETAL = "baremetal" - CLUSTERING = "clustering" - COMPUTE = "compute" - DATA_PROCESSING = "data_processing" - DATABASE = "database" - IDENTITY = "identity" - IMAGE = "image" - MESSAGING = "messaging" - NETWORK = "network" - OBJECT_STORAGE = "object_storage" - ORCHESTRATION = "orchestration" - TELEMETRY = "telemetry" - VOLUME = "volume" - - -class _TempestTestsSets(utils.ImmutableMixin, utils.EnumMixin): - FULL = "full" - SMOKE = "smoke" - SCENARIO = "scenario" - JSON_SCHEMA = "http://json-schema.org/draft-04/schema" @@ -239,8 +218,6 @@ EndpointPermission = _EndpointPermission() ServiceType = _ServiceType() Service = _Service() EndpointType = _EndpointType() -TempestTestsAPI = _TempestTestsAPI() -TempestTestsSets = _TempestTestsSets() HookStatus = _HookStatus() TagType = _TagType() VerifierStatus = _VerifierStatus() diff --git a/rally/plugins/openstack/verification/__init__.py b/rally/plugins/openstack/verification/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally/plugins/openstack/verification/tempest/__init__.py b/rally/plugins/openstack/verification/tempest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally/plugins/openstack/verification/tempest/config.ini b/rally/plugins/openstack/verification/tempest/config.ini new file mode 100644 index 00000000..19459672 --- /dev/null +++ b/rally/plugins/openstack/verification/tempest/config.ini @@ -0,0 +1,55 @@ +[DEFAULT] +debug = True +log_file = tempest.log +use_stderr = False + +[auth] +use_dynamic_credentials = True + +[compute] +image_ref = +image_ref_alt = +flavor_ref = +flavor_ref_alt = +fixed_network_name = + +[compute-feature-enabled] +live_migration = False +resize = True +vnc_console = True +attach_encrypted_volume = False + +[data-processing] + +[identity] + +[image-feature-enabled] +deactivate_image = True + +[input-scenario] +ssh_user_regex = [["^.*[Cc]irros.*$", "cirros"], ["^.*[Tt]est[VvMm].*$", "cirros"], ["^.*rally_verify.*$", "cirros"]] + +[network] + +[network-feature-enabled] +ipv6_subnet_attributes = True +ipv6 = True + +[object-storage] + +[oslo_concurrency] + +[orchestration] +instance_type = + +[scenario] +img_file = + +[service_available] + +[validation] +run_validation = True +image_ssh_user = cirros + +[volume-feature-enabled] +bootable = True diff --git a/rally/plugins/openstack/verification/tempest/config.py b/rally/plugins/openstack/verification/tempest/config.py new file mode 100644 index 00000000..2e934fc7 --- /dev/null +++ b/rally/plugins/openstack/verification/tempest/config.py @@ -0,0 +1,594 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect +import os +import re + +from oslo_config import cfg +import requests +import six +from six.moves import configparser +from six.moves.urllib import parse + +from rally.common.i18n import _ +from rally.common import logging +from rally.common import objects +from rally.common import utils +from rally import exceptions +from rally import osclients +from rally.plugins.openstack.wrappers import glance +from rally.plugins.openstack.wrappers import network +from rally.task import utils as task_utils +from rally.verification import context + + +LOG = logging.getLogger(__name__) + + +TEMPEST_OPTS = [ + cfg.StrOpt("img_url", + deprecated_opts=[cfg.DeprecatedOpt("cirros_img_url", + group="image")], + default="http://download.cirros-cloud.net/" + "0.3.4/cirros-0.3.4-x86_64-disk.img", + help="image URL"), + cfg.StrOpt("img_disk_format", + deprecated_opts=[cfg.DeprecatedOpt("disk_format", + group="image")], + default="qcow2", + help="Image disk format to use when creating the image"), + cfg.StrOpt("img_container_format", + deprecated_opts=[cfg.DeprecatedOpt("container_format", + group="image")], + default="bare", + help="Image container format to use when creating the image"), + cfg.StrOpt("img_name_regex", + deprecated_opts=[cfg.DeprecatedOpt("name_regex", + group="image")], + default="^.*(cirros|testvm).*$", + help="Regular expression for name of a public image to " + "discover it in the cloud and use it for the tests. " + "Note that when Rally is searching for the image, case " + "insensitive matching is performed. Specify nothing " + "('img_name_regex =') if you want to disable discovering. " + "In this case Rally will create needed resources by " + "itself if the values for the corresponding config " + "options are not specified in the Tempest config file"), + cfg.StrOpt("swift_operator_role", + deprecated_group="role", + default="Member", + help="Role required for users " + "to be able to create Swift containers"), + cfg.StrOpt("swift_reseller_admin_role", + deprecated_group="role", + default="ResellerAdmin", + help="User role that has reseller admin"), + cfg.StrOpt("heat_stack_owner_role", + deprecated_group="role", + default="heat_stack_owner", + help="Role required for users " + "to be able to manage Heat stacks"), + cfg.StrOpt("heat_stack_user_role", + deprecated_group="role", + default="heat_stack_user", + help="Role for Heat template-defined users"), + cfg.IntOpt("flavor_ref_ram", + default="64", + help="Primary flavor RAM size used by most of the test cases"), + cfg.IntOpt("flavor_ref_alt_ram", + default="128", + help="Alternate reference flavor RAM size used by test that" + "need two flavors, like those that resize an instance"), + cfg.IntOpt("heat_instance_type_ram", + default="64", + help="RAM size flavor used for orchestration test cases") +] + +CONF = cfg.CONF +CONF.register_opts(TEMPEST_OPTS, "tempest") +CONF.import_opt("glance_image_delete_timeout", + "rally.plugins.openstack.scenarios.glance.utils", + "benchmark") +CONF.import_opt("glance_image_delete_poll_interval", + "rally.plugins.openstack.scenarios.glance.utils", + "benchmark") + + +def _create_or_get_data_dir(): + data_dir = os.path.join( + os.path.expanduser("~"), ".rally", "verification", "tempest", "data") + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + return data_dir + + +def _download_image(image_path, image=None): + if image: + LOG.debug("Downloading image '%s' " + "from Glance to %s" % (image.name, image_path)) + with open(image_path, "wb") as image_file: + for chunk in image.data(): + image_file.write(chunk) + else: + LOG.debug("Downloading image from %s " + "to %s" % (CONF.tempest.img_url, image_path)) + try: + response = requests.get(CONF.tempest.img_url, stream=True) + except requests.ConnectionError as err: + msg = _("Failed to download image. " + "Possibly there is no connection to Internet. " + "Error: %s.") % (str(err) or "unknown") + raise exceptions.TempestConfigCreationFailure(msg) + + if response.status_code == 200: + with open(image_path, "wb") as image_file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + image_file.write(chunk) + image_file.flush() + else: + if response.status_code == 404: + msg = _("Failed to download image. Image was not found.") + else: + msg = _("Failed to download image. " + "HTTP error code %d.") % response.status_code + raise exceptions.TempestConfigCreationFailure(msg) + + LOG.debug("The image has been successfully downloaded!") + + +def write_configfile(path, conf_object): + with open(path, "w") as configfile: + conf_object.write(configfile) + + +def read_configfile(path): + with open(path) as f: + return f.read() + + +def add_extra_options(extra_options, conf_object): + for section in extra_options: + if section not in (conf_object.sections() + ["DEFAULT"]): + conf_object.add_section(section) + for option, value in extra_options[section].items(): + conf_object.set(section, option, value) + + return conf_object + + +def extend_configfile(configfile, extra_options): + conf = configparser.ConfigParser() + + conf.read(configfile) + add_extra_options(extra_options, conf) + write_configfile(configfile, conf) + raw_conf = six.StringIO() + conf.write(raw_conf) + return raw_conf.getvalue() + + +class TempestConfigfileManager(utils.RandomNameGeneratorMixin): + """Class to create a Tempest config file.""" + + def __init__(self, deployment): + self.deployment = deployment + + self.credential = deployment["admin"] + self.clients = osclients.Clients(objects.Credential(**self.credential)) + self.keystone = self.clients.verified_keystone() + self.available_services = self.clients.services().values() + + self.data_dir = _create_or_get_data_dir() + + self.conf = configparser.ConfigParser() + self.conf.read(os.path.join(os.path.dirname(__file__), "config.ini")) + + def _get_service_url(self, service_name): + s_type = self._get_service_type_by_service_name(service_name) + available_endpoints = self.keystone.service_catalog.get_endpoints() + service_endpoints = available_endpoints.get(s_type, []) + for endpoint in service_endpoints: + # If endpoints were returned by Keystone API V2 + if "publicURL" in endpoint: + return endpoint["publicURL"] + # If endpoints were returned by Keystone API V3 + if endpoint["interface"] == "public": + return endpoint["url"] + + def _get_service_type_by_service_name(self, service_name): + for s_type, s_name in self.clients.services().items(): + if s_name == service_name: + return s_type + + def _configure_auth(self, section_name="auth"): + self.conf.set(section_name, "admin_username", + self.credential["username"]) + self.conf.set(section_name, "admin_password", + self.credential["password"]) + self.conf.set(section_name, "admin_project_name", + self.credential["tenant_name"]) + # Keystone v3 related parameter + self.conf.set(section_name, "admin_domain_name", + self.credential["user_domain_name"] or "Default") + + # Sahara has two service types: 'data_processing' and 'data-processing'. + # 'data_processing' is deprecated, but it can be used in previous OpenStack + # releases. So we need to configure the 'catalog_type' option to support + # environments where 'data_processing' is used as service type for Sahara. + def _configure_data_processing(self, section_name="data-processing"): + if "sahara" in self.available_services: + self.conf.set(section_name, "catalog_type", + self._get_service_type_by_service_name("sahara")) + + def _configure_identity(self, section_name="identity"): + self.conf.set(section_name, "region", + self.credential["region_name"]) + + auth_url = self.credential["auth_url"] + if "/v2" not in auth_url and "/v3" not in auth_url: + auth_version = "v2" + auth_url_v2 = parse.urljoin(auth_url, "/v2.0") + else: + url_path = parse.urlparse(auth_url).path + auth_version = url_path[1:3] + auth_url_v2 = auth_url.replace(url_path, "/v2.0") + self.conf.set(section_name, "auth_version", auth_version) + self.conf.set(section_name, "uri", auth_url_v2) + self.conf.set(section_name, "uri_v3", + auth_url_v2.replace("/v2.0", "/v3")) + + self.conf.set(section_name, "disable_ssl_certificate_validation", + str(self.credential["https_insecure"])) + self.conf.set(section_name, "ca_certificates_file", + self.credential["https_cacert"]) + + # The compute section is configured in context class for Tempest resources. + # Options which are configured there: 'image_ref', 'image_ref_alt', + # 'flavor_ref', 'flavor_ref_alt'. + + def _configure_network(self, section_name="network"): + if "neutron" in self.available_services: + neutronclient = self.clients.neutron() + public_nets = [net for net + in neutronclient.list_networks()["networks"] + if net["status"] == "ACTIVE" and + net["router:external"] is True] + if public_nets: + net_id = public_nets[0]["id"] + self.conf.set(section_name, "public_network_id", net_id) + else: + novaclient = self.clients.nova() + net_name = next(net.human_id for net in novaclient.networks.list() + if net.human_id is not None) + self.conf.set("compute", "fixed_network_name", net_name) + self.conf.set("validation", "network_for_ssh", net_name) + + def _configure_network_feature_enabled( + self, section_name="network-feature-enabled"): + if "neutron" in self.available_services: + neutronclient = self.clients.neutron() + extensions = neutronclient.list_ext("extensions", "/extensions", + retrieve_all=True) + aliases = [ext["alias"] for ext in extensions["extensions"]] + aliases_str = ",".join(aliases) + self.conf.set(section_name, "api_extensions", aliases_str) + + def _configure_oslo_concurrency(self, section_name="oslo_concurrency"): + lock_path = os.path.join(self.data_dir, + "lock_files_%s" % self.deployment["uuid"]) + if not os.path.exists(lock_path): + os.makedirs(lock_path) + self.conf.set(section_name, "lock_path", lock_path) + + def _configure_object_storage(self, section_name="object-storage"): + self.conf.set(section_name, "operator_role", + CONF.tempest.swift_operator_role) + self.conf.set(section_name, "reseller_admin_role", + CONF.tempest.swift_reseller_admin_role) + + def _configure_scenario(self, section_name="scenario"): + self.conf.set(section_name, "img_dir", self.data_dir) + + def _configure_service_available(self, section_name="service_available"): + services = ["cinder", "glance", "heat", "ironic", "neutron", "nova", + "sahara", "swift"] + for service in services: + # Convert boolean to string because ConfigParser fails + # on attempt to get option with boolean value + self.conf.set(section_name, service, + str(service in self.available_services)) + + def _configure_validation(self, section_name="validation"): + if "neutron" in self.available_services: + self.conf.set(section_name, "connect_method", "floating") + else: + self.conf.set(section_name, "connect_method", "fixed") + + def _configure_orchestration(self, section_name="orchestration"): + self.conf.set(section_name, "stack_owner_role", + CONF.tempest.heat_stack_owner_role) + self.conf.set(section_name, "stack_user_role", + CONF.tempest.heat_stack_user_role) + + def create(self, conf_path, extra_options=None): + for name, method in inspect.getmembers(self, inspect.ismethod): + if name.startswith("_configure_"): + method() + + if extra_options: + add_extra_options(extra_options, self.conf) + + write_configfile(conf_path, self.conf) + + return read_configfile(conf_path) + + +@context.configure("tempest_configuration", order=900) +class TempestResourcesContext(context.VerifierContext): + """Context class to create/delete resources needed for Tempest.""" + + RESOURCE_NAME_FORMAT = "rally_verify_XXXXXXXX_XXXXXXXX" + + def __init__(self, ctx): + super(TempestResourcesContext, self).__init__(ctx) + credential = self.verifier.deployment["admin"] + self.clients = osclients.Clients(objects.Credential(**credential)) + + self.conf = configparser.ConfigParser() + self.conf_path = self.verifier.manager.configfile + self.data_dir = os.path.join(os.path.expanduser("~"), ".rally", + "verification", "tempest", "data") + self.image_name = "tempest-image" + + self._created_roles = [] + self._created_images = [] + self._created_flavors = [] + self._created_networks = [] + + def setup(self): + self.available_services = self.clients.services().values() + + self.conf.read(self.conf_path) + + self.data_dir = _create_or_get_data_dir() + + self._create_tempest_roles() + + self._configure_option("scenario", "img_file", self.image_name, + helper_method=self._download_image) + self._configure_option("compute", "image_ref", + helper_method=self._discover_or_create_image) + self._configure_option("compute", "image_ref_alt", + helper_method=self._discover_or_create_image) + self._configure_option("compute", "flavor_ref", + helper_method=self._discover_or_create_flavor, + flv_ram=CONF.tempest.flavor_ref_ram) + self._configure_option("compute", "flavor_ref_alt", + helper_method=self._discover_or_create_flavor, + flv_ram=CONF.tempest.flavor_ref_alt_ram) + if "neutron" in self.available_services: + neutronclient = self.clients.neutron() + if neutronclient.list_networks(shared=True)["networks"]: + # If the OpenStack cloud has some shared networks, we will + # create our own shared network and specify its name in the + # Tempest config file. Such approach will allow us to avoid + # failures of Tempest tests with error "Multiple possible + # networks found". Otherwise the default behavior defined in + # Tempest will be used and Tempest itself will manage network + # resources. + LOG.debug("Shared networks found. " + "'fixed_network_name' option should be configured") + self._configure_option( + "compute", "fixed_network_name", + helper_method=self._create_network_resources) + if "heat" in self.available_services: + self._configure_option( + "orchestration", "instance_type", + helper_method=self._discover_or_create_flavor, + flv_ram=CONF.tempest.heat_instance_type_ram) + + write_configfile(self.conf_path, self.conf) + + def cleanup(self): + # Tempest tests may take more than 1 hour and we should remove all + # cached clients sessions to avoid tokens expiration when deleting + # Tempest resources. + self.clients.clear() + + self._cleanup_tempest_roles() + self._cleanup_images() + self._cleanup_flavors() + if "neutron" in self.available_services: + self._cleanup_network_resources() + + write_configfile(self.conf_path, self.conf) + + def _create_tempest_roles(self): + keystoneclient = self.clients.verified_keystone() + roles = [CONF.tempest.swift_operator_role, + CONF.tempest.swift_reseller_admin_role, + CONF.tempest.heat_stack_owner_role, + CONF.tempest.heat_stack_user_role] + existing_roles = set(role.name for role in keystoneclient.roles.list()) + + for role in roles: + if role not in existing_roles: + LOG.debug("Creating role '%s'" % role) + self._created_roles.append(keystoneclient.roles.create(role)) + + def _discover_image(self): + LOG.debug("Trying to discover a public image with name matching " + "regular expression '%s'. Note that case insensitive " + "matching is performed." % CONF.tempest.img_name_regex) + glance_wrapper = glance.wrap(self.clients.glance, self) + images = glance_wrapper.list_images(status="active", + visibility="public") + for image in images: + if image.name and re.match(CONF.tempest.img_name_regex, + image.name, re.IGNORECASE): + LOG.debug("The following public " + "image discovered: '%s'" % image.name) + return image + + LOG.debug("There is no public image with name matching " + "regular expression '%s'" % CONF.tempest.img_name_regex) + + def _download_image(self): + image_path = os.path.join(self.data_dir, self.image_name) + if os.path.isfile(image_path): + LOG.debug("Image is already downloaded to %s" % image_path) + return + + if CONF.tempest.img_name_regex: + image = self._discover_image() + if image: + return _download_image(image_path, image) + + _download_image(image_path) + + def _configure_option(self, section, option, value=None, + helper_method=None, *args, **kwargs): + option_value = self.conf.get(section, option) + if not option_value: + LOG.debug("Option '%s' from '%s' section " + "is not configured" % (option, section)) + if helper_method: + res = helper_method(*args, **kwargs) + if res: + value = res["name"] if "network" in option else res.id + LOG.debug("Setting value '%s' for option '%s'" % (value, option)) + self.conf.set(section, option, value) + LOG.debug("Option '{opt}' is configured. " + "{opt} = {value}".format(opt=option, value=value)) + else: + LOG.debug("Option '{opt}' is already configured " + "in Tempest config file. {opt} = {opt_val}" + .format(opt=option, opt_val=option_value)) + + def _discover_or_create_image(self): + if CONF.tempest.img_name_regex: + image = self._discover_image() + if image: + LOG.debug("Using image '%s' (ID = %s) " + "for the tests" % (image.name, image.id)) + return image + + params = { + "name": self.generate_random_name(), + "disk_format": CONF.tempest.img_disk_format, + "container_format": CONF.tempest.img_container_format, + "image_location": os.path.join(self.data_dir, self.image_name), + "visibility": "public" + } + LOG.debug("Creating image '%s'" % params["name"]) + glance_wrapper = glance.wrap(self.clients.glance, self) + image = glance_wrapper.create_image(**params) + LOG.debug("Image '%s' (ID = %s) has been " + "successfully created!" % (image.name, image.id)) + self._created_images.append(image) + + return image + + def _discover_or_create_flavor(self, flv_ram): + novaclient = self.clients.nova() + + LOG.debug("Trying to discover a flavor with the following " + "properties: RAM = %dMB, VCPUs = 1, disk = 0GB" % flv_ram) + for flavor in novaclient.flavors.list(): + if (flavor.ram == flv_ram and + flavor.vcpus == 1 and flavor.disk == 0): + LOG.debug("The following flavor discovered: '{0}'. " + "Using flavor '{0}' (ID = {1}) for the tests" + .format(flavor.name, flavor.id)) + return flavor + + LOG.debug("There is no flavor with the mentioned properties") + + params = { + "name": self.generate_random_name(), + "ram": flv_ram, + "vcpus": 1, + "disk": 0 + } + LOG.debug("Creating flavor '%s' with the following properties: RAM " + "= %dMB, VCPUs = 1, disk = 0GB" % (params["name"], flv_ram)) + flavor = novaclient.flavors.create(**params) + LOG.debug("Flavor '%s' (ID = %s) has been " + "successfully created!" % (flavor.name, flavor.id)) + self._created_flavors.append(flavor) + + return flavor + + def _create_network_resources(self): + neutron_wrapper = network.NeutronWrapper(self.clients, self) + tenant_id = self.clients.keystone.auth_ref.project_id + LOG.debug("Creating network resources: network, subnet, router") + net = neutron_wrapper.create_network( + tenant_id, subnets_num=1, add_router=True, + network_create_args={"shared": True}) + LOG.debug("Network resources have been successfully created!") + self._created_networks.append(net) + + return net + + def _cleanup_tempest_roles(self): + keystoneclient = self.clients.keystone() + for role in self._created_roles: + LOG.debug("Deleting role '%s'" % role.name) + keystoneclient.roles.delete(role.id) + LOG.debug("Role '%s' has been deleted" % role.name) + + def _cleanup_images(self): + glance_wrapper = glance.wrap(self.clients.glance, self) + for image in self._created_images: + LOG.debug("Deleting image '%s'" % image.name) + self.clients.glance().images.delete(image.id) + task_utils.wait_for_status( + image, ["deleted", "pending_delete"], + check_deletion=True, + update_resource=glance_wrapper.get_image, + timeout=CONF.benchmark.glance_image_delete_timeout, + check_interval=CONF.benchmark. + glance_image_delete_poll_interval) + LOG.debug("Image '%s' has been deleted" % image.name) + self._remove_opt_value_from_config("compute", image.id) + + def _cleanup_flavors(self): + novaclient = self.clients.nova() + for flavor in self._created_flavors: + LOG.debug("Deleting flavor '%s'" % flavor.name) + novaclient.flavors.delete(flavor.id) + LOG.debug("Flavor '%s' has been deleted" % flavor.name) + self._remove_opt_value_from_config("compute", flavor.id) + self._remove_opt_value_from_config("orchestration", flavor.id) + + def _cleanup_network_resources(self): + neutron_wrapper = network.NeutronWrapper(self.clients, self) + for net in self._created_networks: + LOG.debug("Deleting network resources: router, subnet, network") + neutron_wrapper.delete_network(net) + self._remove_opt_value_from_config("compute", net["name"]) + LOG.debug("Network resources have been deleted") + + def _remove_opt_value_from_config(self, section, opt_value): + for option, value in self.conf.items(section): + if opt_value == value: + LOG.debug("Removing value '%s' for option '%s' " + "from Tempest config file" % (opt_value, option)) + self.conf.set(section, option, "") + LOG.debug("Value '%s' has been removed" % opt_value) diff --git a/rally/plugins/openstack/verification/tempest/consts.py b/rally/plugins/openstack/verification/tempest/consts.py new file mode 100644 index 00000000..fa35bc26 --- /dev/null +++ b/rally/plugins/openstack/verification/tempest/consts.py @@ -0,0 +1,46 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.common import utils + + +class _TempestApiTestSets(utils.ImmutableMixin, utils.EnumMixin): + BAREMETAL = "baremetal" + CLUSTERING = "clustering" + COMPUTE = "compute" + DATA_PROCESSING = "data_processing" + DATABASE = "database" + IDENTITY = "identity" + IMAGE = "image" + MESSAGING = "messaging" + NETWORK = "network" + OBJECT_STORAGE = "object_storage" + ORCHESTRATION = "orchestration" + TELEMETRY = "telemetry" + VOLUME = "volume" + + +class _TempestScenarioTestSets(utils.ImmutableMixin, utils.EnumMixin): + SCENARIO = "scenario" + + +class _TempestTestSets(utils.ImmutableMixin, utils.EnumMixin): + FULL = "full" + SMOKE = "smoke" + + +TempestApiTestSets = _TempestApiTestSets() +TempestScenarioTestSets = _TempestScenarioTestSets() +TempestTestSets = _TempestTestSets() diff --git a/rally/plugins/openstack/verification/tempest/manager.py b/rally/plugins/openstack/verification/tempest/manager.py new file mode 100644 index 00000000..d5840a01 --- /dev/null +++ b/rally/plugins/openstack/verification/tempest/manager.py @@ -0,0 +1,178 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import re +import shutil +import subprocess + +import yaml + +from rally.common.i18n import _LE +from rally.common import logging +from rally import exceptions +from rally.plugins.common.verification import testr +from rally.plugins.openstack.verification.tempest import config +from rally.plugins.openstack.verification.tempest import consts +from rally.verification import manager +from rally.verification import utils + + +LOG = logging.getLogger(__name__) + + +@manager.configure(name="tempest", namespace="openstack", + default_repo="https://git.openstack.org/openstack/tempest", + context={"tempest_configuration": {}, + "testr_verifier": {}}) +class TempestManager(testr.TestrLauncher): + """Plugin for Tempest management.""" + + @property + def run_environ(self): + env = super(TempestManager, self).run_environ + env["TEMPEST_CONFIG_DIR"] = os.path.dirname(self.configfile) + env["TEMPEST_CONFIG"] = os.path.basename(self.configfile) + # TODO(andreykurilin): move it to Testr base class + env["OS_TEST_PATH"] = os.path.join(self.repo_dir, + "tempest/test_discover") + return env + + @property + def configfile(self): + return os.path.join(self.home_dir, "tempest.conf") + + def get_configuration(self): + return config.read_configfile(self.configfile) + + def configure(self, extra_options=None): + cm = config.TempestConfigfileManager(self.verifier.deployment) + raw_configfile = cm.create(self.configfile, extra_options) + return raw_configfile + + def extend_configuration(self, extra_options): + return config.extend_configfile(self.configfile, extra_options) + + def override_configuration(self, new_content): + with open(self.configfile, "w") as f: + f.write(new_content) + + def install_extension(self, source, version=None, extra=None): + """Install a Tempest plugin.""" + if extra: + raise NotImplementedError( + _LE("'%s' verifiers don't support extra options for " + "extension installations.") + % self.get_name()) + version = version or "master" + egg = re.sub("\.git$", "", os.path.basename(source.strip("/"))) + full_source = "git+{0}@{1}#egg={2}".format(source, version, egg) + # NOTE(ylobankov): Use 'develop mode' installation to provide an + # ability to advanced users to change tests or + # develop new ones in verifier repo on the fly. + cmd = ["pip", "install", + "--src", os.path.join(self.base_dir, "extensions"), + "-e", full_source] + if self.verifier.system_wide: + cmd.insert(2, "--no-deps") + utils.check_output(cmd, cwd=self.base_dir, env=self.environ) + + # Very often Tempest plugins are inside projects and requirements + # for plugins are listed in the test-requirements.txt file. + test_reqs_path = os.path.join(self.base_dir, "extensions", + egg, "test-requirements.txt") + # TODO(andreykurilin): check that packages from test-requirements are + # present in system in case of system_wide installation + if not self.verifier.system_wide and os.path.exists(test_reqs_path): + utils.check_output(["pip", "install", "-r", test_reqs_path], + cwd=self.base_dir, env=self.environ) + + def list_extensions(self): + """List all installed Tempest plugins.""" + # TODO(andreykurilin): find a better way to list tempest plugins + cmd = ("from tempest.test_discover import plugins; " + "plugins_manager = plugins.TempestTestPluginManager(); " + "plugins_map = plugins_manager.get_plugin_load_tests_tuple(); " + "plugins_list = [" + " {'name': p.name, " + " 'entry_point': p.entry_point_target, " + " 'location': plugins_map[p.name][1]} " + " for p in plugins_manager.ext_plugins.extensions]; " + "print(plugins_list)") + try: + return yaml.load( + utils.check_output(["python", "-c", cmd], cwd=self.base_dir, + env=self.environ).strip()) + except subprocess.CalledProcessError: + raise exceptions.RallyException( + "Cannot list installed Tempest plugins for verifier %s." % + self.verifier) + + def uninstall_extension(self, name): + """Uninstall a Tempest plugin.""" + for ext in self.list_extensions(): + if ext["name"] == name and os.path.exists(ext["location"]): + shutil.rmtree(ext["location"]) + break + else: + raise exceptions.RallyException( + "There is no Tempest plugin with name '%s'. " + "Are you sure that it was installed?" % name) + + def list_tests(self, pattern=""): + """List all tests.""" + if pattern: + pattern = self._transform_pattern(pattern) + return super(TempestManager, self).list_tests(pattern) + + @classmethod + def validate_args(cls, args): + """Validate given arguments.""" + super(TempestManager, cls).validate_args(args) + + if args.get("pattern"): + pattern = args["pattern"].split("=", 1) + if len(pattern) == 1: + pass # it is just a regex + elif pattern[0] == "set": + available_sets = (list(consts.TempestTestSets) + + list(consts.TempestApiTestSets) + + list(consts.TempestScenarioTestSets)) + if pattern[1] not in available_sets: + raise exceptions.ValidationError( + "Test set '%s' not found in available " + "Tempest test sets. Available sets are '%s'." + % (pattern[1], "', '".join(available_sets))) + else: + raise exceptions.ValidationError( + "'pattern' argument should be a regexp or set name " + "(format: 'tempest.api.identity.v3', 'set=smoke').") + + def _transform_pattern(self, pattern): + """Transforms tempest-specific pattern to testr format.""" + parsed_pattern = pattern.split("=", 1) + if len(parsed_pattern) == 2: + if parsed_pattern[0] == "set": + if parsed_pattern[1] in consts.TempestTestSets: + return "smoke" if parsed_pattern[1] == "smoke" else "" + elif parsed_pattern[1] in consts.TempestApiTestSets: + return "tempest.api.%s" % parsed_pattern[1] + else: + return "tempest.%s" % parsed_pattern[1] + return pattern + + def _process_run_args(self, run_args): + if run_args.get("pattern"): + run_args["pattern"] = self._transform_pattern(run_args["pattern"]) + return run_args diff --git a/tests/unit/plugins/openstack/verification/__init__.py b/tests/unit/plugins/openstack/verification/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/plugins/openstack/verification/tempest/__init__.py b/tests/unit/plugins/openstack/verification/tempest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/plugins/openstack/verification/tempest/test_config.py b/tests/unit/plugins/openstack/verification/tempest/test_config.py new file mode 100644 index 00000000..6ab3e752 --- /dev/null +++ b/tests/unit/plugins/openstack/verification/tempest/test_config.py @@ -0,0 +1,633 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import ddt +import mock +from oslo_config import cfg +import requests + +from rally import exceptions +from rally.plugins.openstack.verification.tempest import config +from tests.unit import fakes +from tests.unit import test + +CONF = cfg.CONF + + +CREDS = { + "admin": { + "username": "admin", + "tenant_name": "admin", + "password": "admin-12345", + "auth_url": "http://test:5000/v2.0/", + "permission": "admin", + "region_name": "test", + "https_insecure": False, + "https_cacert": "/path/to/cacert/file", + "user_domain_name": "admin", + "project_domain_name": "admin" + }, + "uuid": "fake_deployment" +} + +PATH = "rally.plugins.openstack.verification.tempest.config" + + +@ddt.ddt +class TempestConfigTestCase(test.TestCase): + + def setUp(self): + super(TempestConfigTestCase, self).setUp() + + mock.patch("rally.osclients.Clients").start() + + self.tempest_conf = config.TempestConfigfileManager(CREDS) + + @ddt.data({"publicURL": "test_url"}, + {"interface": "public", "url": "test_url"}) + def test__get_service_url(self, endpoint): + mock_catalog = mock.MagicMock() + mock_catalog.get_endpoints.return_value = { + "test_service_type": [endpoint]} + + self.tempest_conf.keystone.service_catalog = mock_catalog + self.tempest_conf.clients.services.return_value = { + "test_service_type": "test_service"} + self.assertEqual( + self.tempest_conf._get_service_url("test_service"), "test_url") + + def test__configure_auth(self): + self.tempest_conf._configure_auth() + + expected = ( + ("admin_username", CREDS["admin"]["username"]), + ("admin_password", CREDS["admin"]["password"]), + ("admin_project_name", CREDS["admin"]["tenant_name"]), + ("admin_domain_name", CREDS["admin"]["user_domain_name"])) + result = self.tempest_conf.conf.items("auth") + for item in expected: + self.assertIn(item, result) + + @ddt.data("data_processing", "data-processing") + def test__configure_data_processing(self, service_type): + self.tempest_conf.available_services = ["sahara"] + + self.tempest_conf.clients.services.return_value = { + service_type: "sahara"} + self.tempest_conf._configure_data_processing() + self.assertEqual( + self.tempest_conf.conf.get( + "data-processing", "catalog_type"), service_type) + + def test__configure_identity(self): + self.tempest_conf._configure_identity() + + expected = ( + ("region", CREDS["admin"]["region_name"]), + ("auth_version", "v2"), + ("uri", CREDS["admin"]["auth_url"][:-1]), + ("uri_v3", CREDS["admin"]["auth_url"].replace("/v2.0/", "/v3")), + ("disable_ssl_certificate_validation", + str(CREDS["admin"]["https_insecure"])), + ("ca_certificates_file", CREDS["admin"]["https_cacert"])) + result = self.tempest_conf.conf.items("identity") + for item in expected: + self.assertIn(item, result) + + def test__configure_network_if_neutron(self): + self.tempest_conf.available_services = ["neutron"] + client = self.tempest_conf.clients.neutron() + client.list_networks.return_value = { + "networks": [ + { + "status": "ACTIVE", + "id": "test_id", + "router:external": True + } + ] + } + + self.tempest_conf._configure_network() + self.assertEqual( + self.tempest_conf.conf.get("network", + "public_network_id"), "test_id") + + def test__configure_network_if_nova(self): + self.tempest_conf.available_services = ["nova"] + client = self.tempest_conf.clients.nova() + client.networks.list.return_value = [ + mock.MagicMock(human_id="fake-network")] + + self.tempest_conf._configure_network() + + expected = {"compute": ("fixed_network_name", "fake-network"), + "validation": ("network_for_ssh", "fake-network")} + for section, option in expected.items(): + result = self.tempest_conf.conf.items(section) + self.assertIn(option, result) + + def test__configure_network_feature_enabled(self): + self.tempest_conf.available_services = ["neutron"] + client = self.tempest_conf.clients.neutron() + client.list_ext.return_value = { + "extensions": [ + {"alias": "dvr"}, + {"alias": "extra_dhcp_opt"}, + {"alias": "extraroute"} + ] + } + + self.tempest_conf._configure_network_feature_enabled() + client.list_ext.assert_called_once_with("extensions", "/extensions", + retrieve_all=True) + self.assertEqual(self.tempest_conf.conf.get( + "network-feature-enabled", "api_extensions"), + "dvr,extra_dhcp_opt,extraroute") + + @mock.patch("os.makedirs") + @mock.patch("os.path.exists", return_value=False) + def test__configure_oslo_concurrency(self, mock_exists, mock_makedirs): + self.tempest_conf._configure_oslo_concurrency() + + lock_path = os.path.join( + self.tempest_conf.data_dir, "lock_files_fake_deployment") + mock_makedirs.assert_called_with(lock_path) + self.assertEqual( + self.tempest_conf.conf.get( + "oslo_concurrency", "lock_path"), lock_path) + + def test__configure_object_storage(self): + self.tempest_conf._configure_object_storage() + + expected = ( + ("operator_role", CONF.tempest.swift_operator_role), + ("reseller_admin_role", CONF.tempest.swift_reseller_admin_role)) + result = self.tempest_conf.conf.items("object-storage") + for item in expected: + self.assertIn(item, result) + + def test__configure_orchestration(self): + self.tempest_conf._configure_orchestration() + + expected = ( + ("stack_owner_role", CONF.tempest.heat_stack_owner_role), + ("stack_user_role", CONF.tempest.heat_stack_user_role)) + result = self.tempest_conf.conf.items("orchestration") + for item in expected: + self.assertIn(item, result) + + def test__configure_scenario(self): + self.tempest_conf._configure_scenario() + + expected = (("img_dir", self.tempest_conf.data_dir),) + result = self.tempest_conf.conf.items("scenario") + for item in expected: + self.assertIn(item, result) + + def test__configure_service_available(self): + available_services = ("nova", "cinder", "glance", "sahara") + self.tempest_conf.available_services = available_services + self.tempest_conf._configure_service_available() + + expected = ( + ("neutron", "False"), ("heat", "False"), ("nova", "True"), + ("swift", "False"), ("cinder", "True"), ("sahara", "True"), + ("glance", "True")) + result = self.tempest_conf.conf.items("service_available") + for item in expected: + self.assertIn(item, result) + + @ddt.data({}, {"service": "neutron", "connect_method": "floating"}) + @ddt.unpack + def test__configure_validation(self, service="nova", + connect_method="fixed"): + self.tempest_conf.available_services = [service] + self.tempest_conf._configure_validation() + + expected = (("run_validation", "True"), + ("connect_method", connect_method)) + result = self.tempest_conf.conf.items("validation") + for item in expected: + self.assertIn(item, result) + + @mock.patch("rally.plugins.openstack.verification." + "tempest.config.read_configfile") + @mock.patch("rally.plugins.openstack.verification." + "tempest.config.write_configfile") + @mock.patch("inspect.getmembers") + def test_create(self, mock_inspect_getmembers, mock_write_configfile, + mock_read_configfile): + configure_something_method = mock.MagicMock() + mock_inspect_getmembers.return_value = [("_configure_something", + configure_something_method)] + + fake_extra_conf = {"section": {"option": "value"}} + + self.assertEqual(mock_read_configfile.return_value, + self.tempest_conf.create("/path/to/fake/conf", + fake_extra_conf)) + self.assertEqual(configure_something_method.call_count, 1) + self.assertIn(("option", "value"), + self.tempest_conf.conf.items("section")) + self.assertEqual(mock_write_configfile.call_count, 1) + + +@ddt.ddt +class TempestResourcesContextTestCase(test.TestCase): + + def setUp(self): + super(TempestResourcesContextTestCase, self).setUp() + + mock.patch("rally.osclients.Clients").start() + self.mock_isfile = mock.patch("os.path.isfile", + return_value=True).start() + + cfg = {"verifier": mock.Mock(deployment=CREDS), + "verification": {"uuid": "uuid"}} + cfg["verifier"].manager.configfile = "/fake/path/to/config" + self.context = config.TempestResourcesContext(cfg) + self.context.conf.add_section("compute") + self.context.conf.add_section("orchestration") + self.context.conf.add_section("scenario") + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(), + create=True) + def test__download_image_from_glance(self, mock_open): + self.mock_isfile.return_value = False + img_path = os.path.join(self.context.data_dir, "foo") + img = mock.MagicMock() + img.data.return_value = "data" + + config._download_image(img_path, img) + mock_open.assert_called_once_with(img_path, "wb") + mock_open().write.assert_has_calls([mock.call("d"), + mock.call("a"), + mock.call("t"), + mock.call("a")]) + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open()) + @mock.patch("requests.get", return_value=mock.MagicMock(status_code=200)) + def test__download_image_from_url_success(self, mock_get, mock_open): + self.mock_isfile.return_value = False + img_path = os.path.join(self.context.data_dir, "foo") + mock_get.return_value.iter_content.return_value = "data" + + config._download_image(img_path) + mock_get.assert_called_once_with(CONF.tempest.img_url, stream=True) + mock_open.assert_called_once_with(img_path, "wb") + mock_open().write.assert_has_calls([mock.call("d"), + mock.call("a"), + mock.call("t"), + mock.call("a")]) + + @mock.patch("requests.get") + @ddt.data(404, 500) + def test__download_image_from_url_failure(self, status_code, mock_get): + self.mock_isfile.return_value = False + mock_get.return_value = mock.MagicMock(status_code=status_code) + self.assertRaises( + exceptions.TempestConfigCreationFailure, config._download_image, + os.path.join(self.context.data_dir, "foo")) + + @mock.patch("requests.get", side_effect=requests.ConnectionError()) + def test__download_image_from_url_connection_error( + self, mock_requests_get): + self.mock_isfile.return_value = False + self.assertRaises( + exceptions.TempestConfigCreationFailure, config._download_image, + os.path.join(self.context.data_dir, "foo")) + + @mock.patch("rally.plugins.openstack.wrappers." + "network.NeutronWrapper.create_network") + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open()) + def test_options_configured_manually( + self, mock_open, mock_neutron_wrapper_create_network): + self.context.available_services = ["glance", "heat", "nova", "neutron"] + + self.context.conf.set("compute", "image_ref", "id1") + self.context.conf.set("compute", "image_ref_alt", "id2") + self.context.conf.set("compute", "flavor_ref", "id3") + self.context.conf.set("compute", "flavor_ref_alt", "id4") + self.context.conf.set("compute", "fixed_network_name", "name1") + self.context.conf.set("orchestration", "instance_type", "id5") + self.context.conf.set("scenario", "img_file", "id6") + + self.context.__enter__() + + glanceclient = self.context.clients.glance() + novaclient = self.context.clients.nova() + + self.assertEqual(glanceclient.images.create.call_count, 0) + self.assertEqual(novaclient.flavors.create.call_count, 0) + self.assertEqual(mock_neutron_wrapper_create_network.call_count, 0) + + def test__create_tempest_roles(self): + role1 = CONF.tempest.swift_operator_role + role2 = CONF.tempest.swift_reseller_admin_role + role3 = CONF.tempest.heat_stack_owner_role + role4 = CONF.tempest.heat_stack_user_role + + client = self.context.clients.verified_keystone() + client.roles.list.return_value = [fakes.FakeRole(name=role1), + fakes.FakeRole(name=role2)] + client.roles.create.side_effect = [fakes.FakeFlavor(name=role3), + fakes.FakeFlavor(name=role4)] + + self.context._create_tempest_roles() + self.assertEqual(client.roles.create.call_count, 2) + + created_roles = [role.name for role in self.context._created_roles] + self.assertIn(role3, created_roles) + self.assertIn(role4, created_roles) + + @mock.patch("rally.plugins.openstack.wrappers.glance.wrap") + def test__discover_image(self, mock_wrap): + client = mock_wrap.return_value + client.list_images.return_value = [fakes.FakeImage(name="Foo"), + fakes.FakeImage(name="CirrOS")] + + image = self.context._discover_image() + self.assertEqual("CirrOS", image.name) + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(), + create=True) + @mock.patch("rally.plugins.openstack.wrappers.glance.wrap") + @mock.patch("os.path.isfile", return_value=False) + def test__download_image(self, mock_isfile, mock_wrap, mock_open): + img_1 = mock.MagicMock() + img_1.name = "Foo" + img_2 = mock.MagicMock() + img_2.name = "CirrOS" + img_2.data.return_value = "data" + mock_wrap.return_value.list_images.return_value = [img_1, img_2] + + self.context._download_image() + img_path = os.path.join(self.context.data_dir, self.context.image_name) + mock_open.assert_called_once_with(img_path, "wb") + mock_open().write.assert_has_calls([mock.call("d"), + mock.call("a"), + mock.call("t"), + mock.call("a")]) + + # We can choose any option to test the '_configure_option' method. So let's + # configure the 'flavor_ref' option. + def test__configure_option(self): + helper_method = mock.MagicMock() + helper_method.side_effect = [fakes.FakeFlavor(id="id1")] + + self.context.conf.set("compute", "flavor_ref", "") + self.context._configure_option("compute", "flavor_ref", + helper_method=helper_method, flv_ram=64) + self.assertEqual(helper_method.call_count, 1) + + result = self.context.conf.get("compute", "flavor_ref") + self.assertEqual("id1", result) + + @mock.patch("rally.plugins.openstack.wrappers.glance.wrap") + def test__discover_or_create_image_when_image_exists(self, mock_wrap): + client = mock_wrap.return_value + client.list_images.return_value = [fakes.FakeImage(name="CirrOS")] + + image = self.context._discover_or_create_image() + self.assertEqual("CirrOS", image.name) + self.assertEqual(0, client.create_image.call_count) + self.assertEqual(0, len(self.context._created_images)) + + @mock.patch("rally.plugins.openstack.wrappers.glance.wrap") + def test__discover_or_create_image(self, mock_wrap): + client = mock_wrap.return_value + + image = self.context._discover_or_create_image() + self.assertEqual(image, client.create_image.return_value) + self.assertEqual(self.context._created_images[0], + client.create_image.return_value) + client.create_image.assert_called_once_with( + container_format=CONF.tempest.img_container_format, + image_location=mock.ANY, + disk_format=CONF.tempest.img_disk_format, + name=mock.ANY, + visibility="public") + + def test__discover_or_create_flavor_when_flavor_exists(self): + client = self.context.clients.nova() + client.flavors.list.return_value = [fakes.FakeFlavor(id="id1", ram=64, + vcpus=1, disk=0)] + + flavor = self.context._discover_or_create_flavor(64) + self.assertEqual("id1", flavor.id) + self.assertEqual(0, len(self.context._created_flavors)) + + def test__discover_or_create_flavor(self): + client = self.context.clients.nova() + client.flavors.create.side_effect = [fakes.FakeFlavor(id="id1")] + + flavor = self.context._discover_or_create_flavor(64) + self.assertEqual("id1", flavor.id) + self.assertEqual("id1", self.context._created_flavors[0].id) + + def test__create_network_resources(self): + client = self.context.clients.neutron() + fake_network = { + "id": "nid1", + "name": "network", + "status": "status"} + + client.create_network.side_effect = [{"network": fake_network}] + client.create_router.side_effect = [{"router": {"id": "rid1"}}] + client.create_subnet.side_effect = [{"subnet": {"id": "subid1"}}] + + network = self.context._create_network_resources() + self.assertEqual("nid1", network["id"]) + self.assertEqual("nid1", self.context._created_networks[0]["id"]) + self.assertEqual("rid1", + self.context._created_networks[0]["router_id"]) + self.assertEqual("subid1", + self.context._created_networks[0]["subnets"][0]) + + def test__cleanup_tempest_roles(self): + self.context._created_roles = [fakes.FakeRole(), fakes.FakeRole()] + + self.context._cleanup_tempest_roles() + client = self.context.clients.keystone() + self.assertEqual(client.roles.delete.call_count, 2) + + @mock.patch("rally.plugins.openstack.wrappers.glance.wrap") + def test__cleanup_images(self, mock_wrap): + self.context._created_images = [fakes.FakeImage(id="id1"), + fakes.FakeImage(id="id2")] + + self.context.conf.set("compute", "image_ref", "id1") + self.context.conf.set("compute", "image_ref_alt", "id2") + + wrapper = mock_wrap.return_value + wrapper.get_image.side_effect = [ + fakes.FakeImage(id="id1", status="DELETED"), + fakes.FakeImage(id="id2"), + fakes.FakeImage(id="id2", status="DELETED")] + + self.context._cleanup_images() + client = self.context.clients.glance() + client.images.delete.assert_has_calls([mock.call("id1"), + mock.call("id2")]) + + self.assertEqual("", self.context.conf.get("compute", "image_ref")) + self.assertEqual("", self.context.conf.get("compute", "image_ref_alt")) + + def test__cleanup_flavors(self): + self.context._created_flavors = [fakes.FakeFlavor(id="id1"), + fakes.FakeFlavor(id="id2"), + fakes.FakeFlavor(id="id3")] + + self.context.conf.set("compute", "flavor_ref", "id1") + self.context.conf.set("compute", "flavor_ref_alt", "id2") + self.context.conf.set("orchestration", "instance_type", "id3") + + self.context._cleanup_flavors() + client = self.context.clients.nova() + self.assertEqual(client.flavors.delete.call_count, 3) + + self.assertEqual("", self.context.conf.get("compute", "flavor_ref")) + self.assertEqual("", self.context.conf.get("compute", + "flavor_ref_alt")) + self.assertEqual("", self.context.conf.get("orchestration", + "instance_type")) + + @mock.patch("rally.plugins.openstack.wrappers." + "network.NeutronWrapper.delete_network") + def test__cleanup_network_resources( + self, mock_neutron_wrapper_delete_network): + self.context._created_networks = [{"name": "net-12345"}] + self.context.conf.set("compute", "fixed_network_name", "net-12345") + + self.context._cleanup_network_resources() + self.assertEqual(mock_neutron_wrapper_delete_network.call_count, 1) + self.assertEqual("", self.context.conf.get("compute", + "fixed_network_name")) + + @mock.patch("%s.write_configfile" % PATH) + @mock.patch("%s.TempestResourcesContext._configure_option" % PATH) + @mock.patch("%s.TempestResourcesContext._create_tempest_roles" % PATH) + @mock.patch("%s._create_or_get_data_dir" % PATH) + @mock.patch("%s.osclients.Clients" % PATH) + def test_setup(self, mock_clients, mock__create_or_get_data_dir, + mock__create_tempest_roles, mock__configure_option, + mock_write_configfile): + mock_clients.return_value.services.return_value = {} + verifier = mock.MagicMock(deployment=CREDS) + cfg = config.TempestResourcesContext({"verifier": verifier}) + cfg.conf = mock.Mock() + + # case #1: no neutron and heat + cfg.setup() + + cfg.conf.read.assert_called_once_with(verifier.manager.configfile) + mock__create_or_get_data_dir.assert_called_once_with() + mock__create_tempest_roles.assert_called_once_with() + mock_write_configfile.assert_called_once_with( + verifier.manager.configfile, cfg.conf) + self.assertEqual( + [mock.call("scenario", "img_file", cfg.image_name, + helper_method=cfg._download_image), + mock.call("compute", "image_ref", + helper_method=cfg._discover_or_create_image), + mock.call("compute", "image_ref_alt", + helper_method=cfg._discover_or_create_image), + mock.call("compute", "flavor_ref", + helper_method=cfg._discover_or_create_flavor, + flv_ram=config.CONF.tempest.flavor_ref_ram), + mock.call("compute", "flavor_ref_alt", + helper_method=cfg._discover_or_create_flavor, + flv_ram=config.CONF.tempest.flavor_ref_alt_ram)], + mock__configure_option.call_args_list) + + cfg.conf.reset_mock() + mock__create_or_get_data_dir.reset_mock() + mock__create_tempest_roles.reset_mock() + mock_write_configfile.reset_mock() + mock__configure_option.reset_mock() + + # case #2: neutron and heat are presented + mock_clients.return_value.services.return_value = { + "network": "neutron", "orchestration": "heat"} + cfg.setup() + + cfg.conf.read.assert_called_once_with(verifier.manager.configfile) + mock__create_or_get_data_dir.assert_called_once_with() + mock__create_tempest_roles.assert_called_once_with() + mock_write_configfile.assert_called_once_with( + verifier.manager.configfile, cfg.conf) + self.assertEqual( + [mock.call("scenario", "img_file", cfg.image_name, + helper_method=cfg._download_image), + mock.call("compute", "image_ref", + helper_method=cfg._discover_or_create_image), + mock.call("compute", "image_ref_alt", + helper_method=cfg._discover_or_create_image), + mock.call("compute", "flavor_ref", + helper_method=cfg._discover_or_create_flavor, + flv_ram=config.CONF.tempest.flavor_ref_ram), + mock.call("compute", "flavor_ref_alt", + helper_method=cfg._discover_or_create_flavor, + flv_ram=config.CONF.tempest.flavor_ref_alt_ram), + mock.call("compute", "fixed_network_name", + helper_method=cfg._create_network_resources), + mock.call("orchestration", "instance_type", + helper_method=cfg._discover_or_create_flavor, + flv_ram=config.CONF.tempest.heat_instance_type_ram)], + mock__configure_option.call_args_list) + + +class UtilsTestCase(test.TestCase): + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open()) + def test_write_configfile(self, mock_open): + conf_path = "/path/to/fake/conf" + conf_data = mock.Mock() + + config.write_configfile(conf_path, conf_data) + mock_open.assert_called_once_with(conf_path, "w") + conf_data.write.assert_called_once_with(mock_open.side_effect()) + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open()) + def test_read_configfile(self, mock_open): + conf_path = "/path/to/fake/conf" + + config.read_configfile(conf_path) + mock_open.assert_called_once_with(conf_path) + mock_open.side_effect().read.assert_called_once_with() + + @mock.patch("rally.plugins.openstack.verification.tempest.config." + "six.StringIO") + @mock.patch("rally.plugins.openstack.verification.tempest.config." + "write_configfile") + @mock.patch("rally.plugins.openstack.verification.tempest.config." + "add_extra_options") + @mock.patch("rally.plugins.openstack.verification.tempest.config." + "configparser") + def test_extend_configfile(self, mock_configparser, mock_add_extra_options, + mock_write_configfile, mock_string_io): + conf_path = "/path/to/fake/conf" + extra_options = mock.Mock() + + config.extend_configfile(conf_path, extra_options) + + mock_configparser.ConfigParser.assert_called_once_with() + conf = mock_configparser.ConfigParser.return_value + conf.read.assert_called_once_with(conf_path) + mock_add_extra_options.assert_called_once_with(extra_options, conf) + conf.write.assert_called_once_with(mock_string_io.return_value) + mock_string_io.return_value.getvalue.assert_called_once_with() diff --git a/tests/unit/plugins/openstack/verification/tempest/test_manager.py b/tests/unit/plugins/openstack/verification/tempest/test_manager.py new file mode 100644 index 00000000..42eae5d8 --- /dev/null +++ b/tests/unit/plugins/openstack/verification/tempest/test_manager.py @@ -0,0 +1,233 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os +import subprocess + +import mock + +from rally import exceptions +from rally.plugins.openstack.verification.tempest import manager +from tests.unit import test + + +PATH = "rally.plugins.openstack.verification.tempest.manager" + + +class TempestManagerTestCase(test.TestCase): + def test_run_environ_property(self): + mock.patch("%s.testr.TestrLauncher.run_environ" % PATH, + new={"some": "key"}).start() + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + env = {"some": "key", + "OS_TEST_PATH": os.path.join(tempest.repo_dir, + "tempest/test_discover"), + "TEMPEST_CONFIG": "tempest.conf", + "TEMPEST_CONFIG_DIR": os.path.dirname(tempest.configfile)} + + self.assertEqual(env, tempest.run_environ) + + def test_configfile_property(self): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + self.assertEqual(os.path.join(tempest.home_dir, "tempest.conf"), + tempest.configfile) + + @mock.patch("%s.config.read_configfile" % PATH) + def test_get_configuration(self, mock_read_configfile): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + self.assertEqual(mock_read_configfile.return_value, + tempest.get_configuration()) + mock_read_configfile.assert_called_once_with(tempest.configfile) + + @mock.patch("%s.config.TempestConfigfileManager" % PATH) + def test_configure(self, mock_tempest_configfile_manager): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + cm = mock_tempest_configfile_manager.return_value + extra_options = mock.Mock() + + self.assertEqual(cm.create.return_value, + tempest.configure(extra_options)) + mock_tempest_configfile_manager.assert_called_once_with( + tempest.verifier.deployment) + cm.create.assert_called_once_with(tempest.configfile, extra_options) + + @mock.patch("%s.config.extend_configfile" % PATH) + def test_extend_configuration(self, mock_extend_configfile): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + extra_options = mock.Mock() + self.assertEqual(mock_extend_configfile.return_value, + tempest.extend_configuration(extra_options)) + mock_extend_configfile.assert_called_once_with(tempest.configfile, + extra_options) + + @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open()) + def test_override_configuration(self, mock_open): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + new_content = mock.Mock() + + tempest.override_configuration(new_content) + + mock_open.assert_called_once_with(tempest.configfile, "w") + mock_open.side_effect().write.assert_called_once_with(new_content) + + @mock.patch("%s.os.path.exists" % PATH) + @mock.patch("%s.utils.check_output" % PATH) + def test_install_extension(self, mock_check_output, mock_exists): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd", + system_wide=True)) + e = self.assertRaises(NotImplementedError, tempest.install_extension, + None, None, {"key": "value"}) + self.assertIn("verifiers don't support extra options", "%s" % e) + + # case #1 system-wide installation + source = "https://github.com/example/example" + tempest.install_extension(source) + + path = os.path.join(tempest.base_dir, "extensions") + mock_check_output.assert_called_once_with( + ["pip", "install", "--no-deps", "--src", path, "-e", + "git+https://github.com/example/example@master#egg=example"], + cwd=tempest.base_dir, env=tempest.environ) + self.assertFalse(mock_exists.called) + + mock_check_output.reset_mock() + + # case #2 virtual env with specified version + tempest.verifier.system_wide = False + version = "some" + tempest.install_extension(source, version=version) + + test_reqs_path = os.path.join(tempest.base_dir, "extensions", + "example", "test-requirements.txt") + self.assertEqual([ + mock.call([ + "pip", "install", "--src", path, "-e", + "git+https://github.com/example/example@some#egg=example"], + cwd=tempest.base_dir, env=tempest.environ), + mock.call(["pip", "install", "-r", test_reqs_path], + cwd=tempest.base_dir, env=tempest.environ)], + mock_check_output.call_args_list) + mock_exists.assert_called_once_with(test_reqs_path) + + @mock.patch("%s.utils.check_output" % PATH) + def test_list_extensions(self, mock_check_output): + plugins_list = [ + {"name": "some", "entry_point": "foo.bar", "location": "/tmp"}, + {"name": "another", "entry_point": "bar.foo", "location": "/tmp"} + ] + mock_check_output.return_value = json.dumps(plugins_list) + + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + + self.assertEqual(plugins_list, tempest.list_extensions()) + self.assertEqual(1, mock_check_output.call_count) + mock_check_output.reset_mock() + + mock_check_output.side_effect = subprocess.CalledProcessError("", "") + self.assertRaises(exceptions.RallyException, tempest.list_extensions) + self.assertEqual(1, mock_check_output.call_count) + + @mock.patch("%s.TempestManager.list_extensions" % PATH) + @mock.patch("%s.os.path.exists" % PATH) + @mock.patch("%s.shutil.rmtree" % PATH) + def test_uninstall_extension(self, mock_rmtree, mock_exists, + mock_list_extensions): + plugins_list = [ + {"name": "some", "entry_point": "foo.bar", "location": "/tmp"}, + {"name": "another", "entry_point": "bar.foo", "location": "/tmp"} + ] + mock_list_extensions.return_value = plugins_list + + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + + tempest.uninstall_extension("some") + mock_rmtree.assert_called_once_with(plugins_list[0]["location"]) + mock_list_extensions.assert_called_once_with() + + mock_rmtree.reset_mock() + mock_list_extensions.reset_mock() + + self.assertRaises(exceptions.RallyException, + tempest.uninstall_extension, "unexist") + + mock_list_extensions.assert_called_once_with() + self.assertFalse(mock_rmtree.called) + + @mock.patch("%s.TempestManager._transform_pattern" % PATH) + @mock.patch("%s.testr.TestrLauncher.list_tests" % PATH) + def test_list_tests(self, mock_testr_launcher_list_tests, + mock__transform_pattern): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + + self.assertEqual(mock_testr_launcher_list_tests.return_value, + tempest.list_tests()) + mock_testr_launcher_list_tests.assert_called_once_with("") + self.assertFalse(mock__transform_pattern.called) + mock_testr_launcher_list_tests.reset_mock() + + pattern = mock.Mock() + + self.assertEqual(mock_testr_launcher_list_tests.return_value, + tempest.list_tests(pattern)) + mock_testr_launcher_list_tests.assert_called_once_with( + mock__transform_pattern.return_value) + mock__transform_pattern.assert_called_once_with(pattern) + + @mock.patch("%s.testr.TestrLauncher.validate_args" % PATH) + def test_validate_args(self, mock_testr_launcher_validate_args): + manager.TempestManager.validate_args({}) + manager.TempestManager.validate_args({"pattern": "some.test"}) + manager.TempestManager.validate_args({"pattern": "set=smoke"}) + manager.TempestManager.validate_args({"pattern": "set=compute"}) + manager.TempestManager.validate_args({"pattern": "set=full"}) + + e = self.assertRaises(exceptions.ValidationError, + manager.TempestManager.validate_args, + {"pattern": "foo=bar"}) + self.assertEqual("Validation error: 'pattern' argument should be a " + "regexp or set name (format: 'tempest.api.identity." + "v3', 'set=smoke').", "%s" % e) + + e = self.assertRaises(exceptions.ValidationError, + manager.TempestManager.validate_args, + {"pattern": "set=foo"}) + self.assertIn("Test set 'foo' not found in available Tempest test " + "sets. Available sets are ", "%s" % e) + + def test__transform_pattern(self): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + + self.assertEqual("foo", tempest._transform_pattern("foo")) + self.assertEqual("foo=bar", tempest._transform_pattern("foo=bar")) + self.assertEqual("", tempest._transform_pattern("set=full")) + self.assertEqual("smoke", tempest._transform_pattern("set=smoke")) + self.assertEqual("tempest.bar", tempest._transform_pattern("set=bar")) + self.assertEqual("tempest.api.compute", + tempest._transform_pattern("set=compute")) + + @mock.patch("%s.TempestManager._transform_pattern" % PATH) + def test__process_run_args(self, mock__transform_pattern): + tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd")) + + self.assertEqual({}, tempest._process_run_args({})) + self.assertFalse(mock__transform_pattern.called) + + self.assertEqual({"foo": "bar"}, + tempest._process_run_args({"foo": "bar"})) + self.assertFalse(mock__transform_pattern.called) + + pattern = mock.Mock() + self.assertEqual({"pattern": mock__transform_pattern.return_value}, + tempest._process_run_args({"pattern": pattern})) + mock__transform_pattern.assert_called_once_with(pattern)