From 951c6f5f02be9f263fb18942cfbb7ac2e77a6acf Mon Sep 17 00:00:00 2001 From: Jose Idar Date: Fri, 12 Sep 2014 16:00:08 -0500 Subject: [PATCH] Blockstorage updates * Added Blockstorage Datasets * Added per-volume-type min and max size configurability. * Added behavior method for getting min and max size of configured volume types * Added 'default' min and max volume type properties to the volumes_api config, which deprecate the older min and max volume_size properties. * Added volume_type_properties property in volumes_api config * Added Image, Flavor, and Volume Type filters and filter mode options to the volumes_api config. * Added configuration option for toggling cross-type snapshot restores (since this will soon be configurable in cinder) Change-Id: I8a4ba2de4dda66c61abf14938dfd14635ff68456 --- cloudcafe/blockstorage/composites.py | 2 - cloudcafe/blockstorage/datasets.py | 161 ++++++++++++++++++ .../volumes_api/common/behaviors.py | 23 ++- .../blockstorage/volumes_api/common/config.py | 97 +++++++++-- 4 files changed, 263 insertions(+), 20 deletions(-) create mode 100644 cloudcafe/blockstorage/datasets.py diff --git a/cloudcafe/blockstorage/composites.py b/cloudcafe/blockstorage/composites.py index 4c907622..a97e6a4b 100644 --- a/cloudcafe/blockstorage/composites.py +++ b/cloudcafe/blockstorage/composites.py @@ -42,7 +42,6 @@ class _BaseVolumesComposite(object): self.behaviors = self._behaviors(self.client) -#For version specific tests class VolumesV1Composite(_BaseVolumesComposite): _config = v1Config _client = v1Client @@ -55,7 +54,6 @@ class VolumesV2Composite(_BaseVolumesComposite): _behaviors = v2Behaviors -#For version agnostic tests class VolumesAutoComposite(object): def __new__(cls): config = VolumesAPIConfig() diff --git a/cloudcafe/blockstorage/datasets.py b/cloudcafe/blockstorage/datasets.py new file mode 100644 index 00000000..4292f62b --- /dev/null +++ b/cloudcafe/blockstorage/datasets.py @@ -0,0 +1,161 @@ +""" +Copyright 2014 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cafe.drivers.unittest.datasets import DatasetList +from cafe.drivers.unittest.decorators import memoized + +from cloudcafe.common.datasets import ModelBasedDatasetToolkit +from cloudcafe.blockstorage.composites import VolumesAutoComposite +from cloudcafe.compute.datasets import ComputeDatasets + + +class BlockstorageDatasets(ModelBasedDatasetToolkit): + """Collection of dataset generators for blockstorage data driven tests""" + _volumes = VolumesAutoComposite() + + @classmethod + @memoized + def _get_volume_types(cls): + """Gets list of all Volume Types in the environment, and caches it for + future calls""" + return cls._get_model_list( + cls._volumes.client.list_all_volume_types, 'volume_types') + + @classmethod + @memoized + def volume_types( + cls, max_datasets=None, randomize=False, model_filter=None, + filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE): + """Returns a DatasetList of all VolumeTypes + Filters should be dictionaries with model attributes as keys and + lists of attributes as key values + """ + + volume_type_list = cls._get_volume_types() + volume_type_list = cls._filter_model_list( + volume_type_list, model_filter=model_filter, + filter_mode=filter_mode) + + dataset_list = DatasetList() + for vol_type in volume_type_list: + data = {'volume_type_name': vol_type.name, + 'volume_type_id': vol_type.id_} + dataset_list.append_new_dataset(vol_type.name, data) + + # Apply modifiers + return cls._modify_dataset_list( + dataset_list, max_datasets=max_datasets, randomize=randomize) + + @classmethod + def configured_volume_types(cls, max_datasets=None, randomize=None): + """Returns a DatasetList of permuations of Volume Types and Images. + Requests all available images and volume types from API, and applies + pre-configured image and volume_type filters. + """ + + volume_type_filter = cls.volumes.config.volume_type_filter + volume_type_filter_mode = cls.volumes.config.volume_type_filter_mode + return cls.volume_types( + max_datasets=max_datasets, randomize=randomize, + model_filter=volume_type_filter, + filter_mode=volume_type_filter_mode) + + +class ComputeIntegrationDatasets(ComputeDatasets, BlockstorageDatasets): + + @classmethod + def images_by_volume_type( + cls, max_datasets=None, randomize=False, + image_filter=None, volume_type_filter=None, + image_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE, + volume_type_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE): + """Returns a DatasetList of all combinations of Images and + Volume Types. + Filters should be dictionaries with model attributes as keys and + lists of attributes as key values + """ + image_list = cls._get_images() + image_list = cls._filter_model_list( + image_list, model_filter=image_filter, + filter_mode=image_filter_mode) + + volume_type_list = cls._get_volume_types() + volume_type_list = cls._filter_model_list( + volume_type_list, model_filter=volume_type_filter, + filter_mode=volume_type_filter_mode) + + # Create dataset from all combinations of all images and volume types + dataset_list = DatasetList() + for vtype in volume_type_list: + for image in image_list: + data = {'volume_type': vtype, + 'image': image} + testname = \ + "{0}_and_{1}".format( + str(vtype.name).replace(" ", "_"), + str(image.name).replace(" ", "_")) + dataset_list.append_new_dataset(testname, data) + + # Apply modifiers + return cls._modify_dataset_list( + dataset_list, max_datasets=max_datasets, randomize=randomize) + + @classmethod + def configured_images(cls, max_datasets=None, randomize=None): + """Returns a DatasetList of permuations of Images. + Requests all available images from API, and applies pre-configured + image and volume_type filters. + """ + + image_filter = cls._volumes.config.image_filter + image_filter_mode = cls._volumes.config.image_filter_mode + return cls.images( + max_datasets=max_datasets, randomize=randomize, + model_filter=image_filter, filter_mode=image_filter_mode) + + @classmethod + def configured_images_by_volume_type( + cls, max_datasets=None, randomize=None): + """Returns a DatasetList of permuations of Volume Types and Images. + Requests all available images and volume types from the API, and + applies pre-configured image and volume_type filters. + """ + + image_filter = cls._volumes.config.image_filter + volume_type_filter = cls._volumes.config.volume_type_filter + image_filter_mode = cls._volumes.config.image_filter_mode + volume_type_filter_mode = cls._volumes.config.volume_type_filter_mode + return cls.images_by_volume_type( + max_datasets=max_datasets, randomize=randomize, + image_filter=image_filter, volume_type_filter=volume_type_filter, + image_filter_mode=image_filter_mode, + volume_type_filter_mode=volume_type_filter_mode) + + @classmethod + def configured_images_by_flavor(cls, max_datasets=None, randomize=None): + """Returns a DatasetList of permuations of Images and Flavors. + Requests all available images and flavors from the API, and applies + pre-configured image and flavor filters. + """ + image_filter = cls._volumes.config.image_filter + image_filter_mode = cls._volumes.config.image_filter_mode + flavor_filter = cls._volumes.config.flavor_filter + flavor_filter_mode = cls._volumes.config.flavor_filter_mode + return cls.images_by_flavor( + max_datasets=max_datasets, randomize=randomize, + image_filter=image_filter, flavor_filter=flavor_filter, + image_filter_mode=image_filter_mode, + flavor_filter_mode=flavor_filter_mode) diff --git a/cloudcafe/blockstorage/volumes_api/common/behaviors.py b/cloudcafe/blockstorage/volumes_api/common/behaviors.py index a1d88065..70b7becf 100644 --- a/cloudcafe/blockstorage/volumes_api/common/behaviors.py +++ b/cloudcafe/blockstorage/volumes_api/common/behaviors.py @@ -141,6 +141,27 @@ class VolumesAPI_CommonBehaviors(BaseBehavior): also call create_volume.""" raise NotImplementedError + def get_configured_volume_type_property( + self, configured_property, id_=None, name=None): + configured_data = self.config.volume_type_properties + + # Raise an exception if any of the configured data has null + # values in it + property_names = ["name", "id"] + for entry in configured_data: + for pname in property_names: + if hasattr(entry, pname): + if entry.get(pname) is None: + raise Exception( + "Ambiguous volume type properties: 'null' value " + "found for configured volume type property '{0}'" + .format(pname)) + + if name and str(entry.get('name') == str(name)): + return entry.get(configured_property) + if id_ is not None and str(entry.get('id')) == str(id_): + return entry.get(configured_property) + def get_volume_info(self, volume_id): resp = self.client.get_volume_info(volume_id=volume_id) self._verify_entity(resp) @@ -470,7 +491,7 @@ class VolumesAPI_CommonBehaviors(BaseBehavior): return False return True - def get_volume_types(self): + def get_volume_type_list(self): resp = self.client.list_all_volume_types() self._verify_entity(resp) return resp.entity diff --git a/cloudcafe/blockstorage/volumes_api/common/config.py b/cloudcafe/blockstorage/volumes_api/common/config.py index b1b804fe..bd657ad9 100644 --- a/cloudcafe/blockstorage/volumes_api/common/config.py +++ b/cloudcafe/blockstorage/volumes_api/common/config.py @@ -15,6 +15,8 @@ limitations under the License. """ import json +from warnings import warn + from cloudcafe.common.models.configuration import ConfigSectionInterface @@ -38,22 +40,6 @@ class VolumesAPIConfig(ConfigSectionInterface): """Version of the cinder api under test, either '1' or '2' """ return self.get("version_under_test", default="1") -# Volume and Snapshot behavior config - @property - def default_volume_type(self): - """Sets the default volume type for some behaviors and tests""" - return self.get("default_volume_type") - - @property - def max_volume_size(self): - """Maximum volume size allowed by the environment under test""" - return int(self.get("max_volume_size", default=1024)) - - @property - def min_volume_size(self): - """Minimum volume size allowed by the environment under test""" - return int(self.get("min_volume_size", default=1)) - @property def volume_status_poll_frequency(self): """Controls the rate at which some behaviors will poll the cinder @@ -68,6 +54,54 @@ class VolumesAPIConfig(ConfigSectionInterface): """ return int(self.get("snapshot_status_poll_frequency", default=10)) +# Volume Type configuration + @property + def volume_type_properties(self): + """Dictionary of volume type properties""" + data = self.get( + 'volume_type_properties', + '[{"name":null, "id":null, "min_size":null, "max_size":null}]') + return json.loads(data) + + @property + def default_volume_type(self): + """Sets the default volume type for some non-data-driven tests.""" + return self.get("default_volume_type") + + @property + def default_volume_type_min_size(self): + """The minimum size allowed by the API for the configured + default volume type + """ + return int(self.get("default_volume_type_min_size")) + + @property + def default_volume_type_max_size(self): + """The maximum size allowed by the API for the configured + default volume type + """ + return int(self.get("default_volume_type_max_size")) + + @property + def min_volume_size(self): + """Deprecated. Use default_volume_type_min_size instead""" + warn( + "This config property is deprecated. Please use the config " + "property 'default_volume_type_min_size' or the much more flexible" + " 'volume_type_properties' instead.") + + return int(self.get("min_volume_size")) + + @property + def max_volume_size(self): + """Deprecated. Use default_volume_type_max_size instead""" + warn( + "This config property is deprecated. Please use the config " + "property 'default_volume_type_max_size' or the much more flexible" + " 'volume_type_properties' instead.") + + return int(self.get("max_volume_size")) + # Volume create timeouts @property def volume_create_min_timeout(self): @@ -157,7 +191,8 @@ class VolumesAPIConfig(ConfigSectionInterface): """Minimum size a volume can be if building from an image. Used by some behaviors and tests. Depending on how the environment under test is deployed, this value may be superceded by the - minimum allowed volume size + minimum allowed volume size, and is otherwise dependent on the image + being used for testing. """ return int(self.get("min_volume_from_image_size")) @@ -294,6 +329,24 @@ class VolumesAPIConfig(ConfigSectionInterface): """ return json.loads(self.get('image_filter', '{}')) + @property + def image_filter_mode(self): + return self.get("image_filter_mode", 'inclusion') + + @property + def flavor_filter(self): + """Expects Json. Returns an empty dictionary by default (no filter). + Dictionary keys should be attributes of the flavor model, and key + values should be a list of values for that model attribute. + Used by some tests to decide which flavors to target for a given + test run. + """ + return json.loads(self.get('flavor_filter', '{}')) + + @property + def flavor_filter_mode(self): + return self.get("flavor_filter_mode", 'inclusion') + @property def volume_type_filter(self): """Expects Json. Returns an empty dictionary by default. @@ -303,3 +356,13 @@ class VolumesAPIConfig(ConfigSectionInterface): test run. """ return json.loads(self.get('volume_type_filter', '{}')) + + @property + def volume_type_filter_mode(self): + return self.get("volume_type_filter_mode", 'inclusion') + +# API configuration + @property + def allow_snapshot_restore_to_different_type(self): + return self.get_boolean( + "allow_snapshot_restore_to_different_type", False)