Blockstorage updates

* Added Blockstorage Datasets
* Added per-volume-type min and max size configurability.
* Added behavior method for getting min and max size of
  configured volume types
* Added 'default' min and max volume type properties to the
  volumes_api config, which deprecate the older min and max
  volume_size properties.
* Added volume_type_properties property in volumes_api config
* Added Image, Flavor, and Volume Type filters and filter mode
  options to the volumes_api config.
* Added configuration option for toggling cross-type snapshot
  restores (since this will soon be configurable in cinder)

Change-Id: I8a4ba2de4dda66c61abf14938dfd14635ff68456

cloudcafe/blockstorage/composites.py

@@ -42,7 +42,6 @@ class _BaseVolumesComposite(object):
         self.behaviors = self._behaviors(self.client)
 
 
-#For version specific tests
 class VolumesV1Composite(_BaseVolumesComposite):
     _config = v1Config
     _client = v1Client
@@ -55,7 +54,6 @@ class VolumesV2Composite(_BaseVolumesComposite):
     _behaviors = v2Behaviors
 
 
-#For version agnostic tests
 class VolumesAutoComposite(object):
     def __new__(cls):
         config = VolumesAPIConfig()

cloudcafe/blockstorage/datasets.py

@@ -0,0 +1,161 @@
+"""
+Copyright 2014 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from cafe.drivers.unittest.datasets import DatasetList
+from cafe.drivers.unittest.decorators import memoized
+
+from cloudcafe.common.datasets import ModelBasedDatasetToolkit
+from cloudcafe.blockstorage.composites import VolumesAutoComposite
+from cloudcafe.compute.datasets import ComputeDatasets
+
+
+class BlockstorageDatasets(ModelBasedDatasetToolkit):
+    """Collection of dataset generators for blockstorage data driven tests"""
+    _volumes = VolumesAutoComposite()
+
+    @classmethod
+    @memoized
+    def _get_volume_types(cls):
+        """Gets list of all Volume Types in the environment, and caches it for
+        future calls"""
+        return cls._get_model_list(
+            cls._volumes.client.list_all_volume_types, 'volume_types')
+
+    @classmethod
+    @memoized
+    def volume_types(
+            cls, max_datasets=None, randomize=False, model_filter=None,
+            filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE):
+        """Returns a DatasetList of all VolumeTypes
+        Filters should be dictionaries with model attributes as keys and
+        lists of attributes as key values
+        """
+
+        volume_type_list = cls._get_volume_types()
+        volume_type_list = cls._filter_model_list(
+            volume_type_list, model_filter=model_filter,
+            filter_mode=filter_mode)
+
+        dataset_list = DatasetList()
+        for vol_type in volume_type_list:
+            data = {'volume_type_name': vol_type.name,
+                    'volume_type_id': vol_type.id_}
+            dataset_list.append_new_dataset(vol_type.name, data)
+
+        # Apply modifiers
+        return cls._modify_dataset_list(
+            dataset_list, max_datasets=max_datasets, randomize=randomize)
+
+    @classmethod
+    def configured_volume_types(cls, max_datasets=None, randomize=None):
+        """Returns a DatasetList of permuations of Volume Types and Images.
+        Requests all available images and volume types from API, and applies
+        pre-configured image and volume_type filters.
+        """
+
+        volume_type_filter = cls.volumes.config.volume_type_filter
+        volume_type_filter_mode = cls.volumes.config.volume_type_filter_mode
+        return cls.volume_types(
+            max_datasets=max_datasets, randomize=randomize,
+            model_filter=volume_type_filter,
+            filter_mode=volume_type_filter_mode)
+
+
+class ComputeIntegrationDatasets(ComputeDatasets, BlockstorageDatasets):
+
+    @classmethod
+    def images_by_volume_type(
+            cls, max_datasets=None, randomize=False,
+            image_filter=None, volume_type_filter=None,
+            image_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE,
+            volume_type_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE):
+        """Returns a DatasetList of all combinations of Images and
+        Volume Types.
+        Filters should be dictionaries with model attributes as keys and
+        lists of attributes as key values
+        """
+        image_list = cls._get_images()
+        image_list = cls._filter_model_list(
+            image_list, model_filter=image_filter,
+            filter_mode=image_filter_mode)
+
+        volume_type_list = cls._get_volume_types()
+        volume_type_list = cls._filter_model_list(
+            volume_type_list, model_filter=volume_type_filter,
+            filter_mode=volume_type_filter_mode)
+
+        # Create dataset from all combinations of all images and volume types
+        dataset_list = DatasetList()
+        for vtype in volume_type_list:
+            for image in image_list:
+                data = {'volume_type': vtype,
+                        'image': image}
+                testname = \
+                    "{0}_and_{1}".format(
+                        str(vtype.name).replace(" ", "_"),
+                        str(image.name).replace(" ", "_"))
+                dataset_list.append_new_dataset(testname, data)
+
+        # Apply modifiers
+        return cls._modify_dataset_list(
+            dataset_list, max_datasets=max_datasets, randomize=randomize)
+
+    @classmethod
+    def configured_images(cls, max_datasets=None, randomize=None):
+        """Returns a DatasetList of permuations of Images.
+        Requests all available images from API, and applies pre-configured
+        image and volume_type filters.
+        """
+
+        image_filter = cls._volumes.config.image_filter
+        image_filter_mode = cls._volumes.config.image_filter_mode
+        return cls.images(
+            max_datasets=max_datasets, randomize=randomize,
+            model_filter=image_filter, filter_mode=image_filter_mode)
+
+    @classmethod
+    def configured_images_by_volume_type(
+            cls, max_datasets=None, randomize=None):
+        """Returns a DatasetList of permuations of Volume Types and Images.
+        Requests all available images and volume types from the API, and
+        applies pre-configured image and volume_type filters.
+        """
+
+        image_filter = cls._volumes.config.image_filter
+        volume_type_filter = cls._volumes.config.volume_type_filter
+        image_filter_mode = cls._volumes.config.image_filter_mode
+        volume_type_filter_mode = cls._volumes.config.volume_type_filter_mode
+        return cls.images_by_volume_type(
+            max_datasets=max_datasets, randomize=randomize,
+            image_filter=image_filter, volume_type_filter=volume_type_filter,
+            image_filter_mode=image_filter_mode,
+            volume_type_filter_mode=volume_type_filter_mode)
+
+    @classmethod
+    def configured_images_by_flavor(cls, max_datasets=None, randomize=None):
+        """Returns a DatasetList of permuations of Images and Flavors.
+        Requests all available images and flavors from the API, and applies
+        pre-configured image and flavor filters.
+        """
+        image_filter = cls._volumes.config.image_filter
+        image_filter_mode = cls._volumes.config.image_filter_mode
+        flavor_filter = cls._volumes.config.flavor_filter
+        flavor_filter_mode = cls._volumes.config.flavor_filter_mode
+        return cls.images_by_flavor(
+            max_datasets=max_datasets, randomize=randomize,
+            image_filter=image_filter, flavor_filter=flavor_filter,
+            image_filter_mode=image_filter_mode,
+            flavor_filter_mode=flavor_filter_mode)

cloudcafe/blockstorage/volumes_api/common/behaviors.py

@@ -141,6 +141,27 @@ class VolumesAPI_CommonBehaviors(BaseBehavior):
         also call create_volume."""
         raise NotImplementedError
 
+    def get_configured_volume_type_property(
+            self, configured_property, id_=None, name=None):
+        configured_data = self.config.volume_type_properties
+
+        # Raise an exception if any of the configured data has null
+        # values in it
+        property_names = ["name", "id"]
+        for entry in configured_data:
+            for pname in property_names:
+                if hasattr(entry, pname):
+                    if entry.get(pname) is None:
+                        raise Exception(
+                            "Ambiguous volume type properties: 'null' value "
+                            "found for configured volume type property '{0}'"
+                            .format(pname))
+
+            if name and str(entry.get('name') == str(name)):
+                return entry.get(configured_property)
+            if id_ is not None and str(entry.get('id')) == str(id_):
+                return entry.get(configured_property)
+
     def get_volume_info(self, volume_id):
         resp = self.client.get_volume_info(volume_id=volume_id)
         self._verify_entity(resp)
@@ -470,7 +491,7 @@ class VolumesAPI_CommonBehaviors(BaseBehavior):
             return False
         return True
 
-    def get_volume_types(self):
+    def get_volume_type_list(self):
         resp = self.client.list_all_volume_types()
         self._verify_entity(resp)
         return resp.entity

cloudcafe/blockstorage/volumes_api/common/config.py

@@ -15,6 +15,8 @@ limitations under the License.
 """
 
 import json
+from warnings import warn
+
 from cloudcafe.common.models.configuration import ConfigSectionInterface
 
 
@@ -38,22 +40,6 @@ class VolumesAPIConfig(ConfigSectionInterface):
         """Version of the cinder api under test, either '1'  or '2' """
         return self.get("version_under_test", default="1")
 
-# Volume and Snapshot behavior config
-    @property
-    def default_volume_type(self):
-        """Sets the default volume type for some behaviors and tests"""
-        return self.get("default_volume_type")
-
-    @property
-    def max_volume_size(self):
-        """Maximum volume size allowed by the environment under test"""
-        return int(self.get("max_volume_size", default=1024))
-
-    @property
-    def min_volume_size(self):
-        """Minimum volume size allowed by the environment under test"""
-        return int(self.get("min_volume_size", default=1))
-
     @property
     def volume_status_poll_frequency(self):
         """Controls the rate at which some behaviors will poll the cinder
@@ -68,6 +54,54 @@ class VolumesAPIConfig(ConfigSectionInterface):
         """
         return int(self.get("snapshot_status_poll_frequency", default=10))
 
+# Volume Type configuration
+    @property
+    def volume_type_properties(self):
+        """Dictionary of volume type properties"""
+        data = self.get(
+            'volume_type_properties',
+            '[{"name":null, "id":null, "min_size":null, "max_size":null}]')
+        return json.loads(data)
+
+    @property
+    def default_volume_type(self):
+        """Sets the default volume type for some non-data-driven tests."""
+        return self.get("default_volume_type")
+
+    @property
+    def default_volume_type_min_size(self):
+        """The minimum size allowed by the API for the configured
+        default volume type
+        """
+        return int(self.get("default_volume_type_min_size"))
+
+    @property
+    def default_volume_type_max_size(self):
+        """The maximum size allowed by the API for the configured
+        default volume type
+        """
+        return int(self.get("default_volume_type_max_size"))
+
+    @property
+    def min_volume_size(self):
+        """Deprecated.  Use default_volume_type_min_size instead"""
+        warn(
+            "This config property is deprecated. Please use the config "
+            "property 'default_volume_type_min_size' or the much more flexible"
+            " 'volume_type_properties' instead.")
+
+        return int(self.get("min_volume_size"))
+
+    @property
+    def max_volume_size(self):
+        """Deprecated.  Use default_volume_type_max_size instead"""
+        warn(
+            "This config property is deprecated. Please use the config "
+            "property 'default_volume_type_max_size' or the much more flexible"
+            " 'volume_type_properties' instead.")
+
+        return int(self.get("max_volume_size"))
+
 # Volume create timeouts
     @property
     def volume_create_min_timeout(self):
@@ -157,7 +191,8 @@ class VolumesAPIConfig(ConfigSectionInterface):
         """Minimum size a volume can be if building from an image.
         Used by some behaviors and tests.  Depending on how the environment
         under test is deployed, this value may be superceded by the
-        minimum allowed volume size
+        minimum allowed volume size, and is otherwise dependent on the image
+        being used for testing.
         """
         return int(self.get("min_volume_from_image_size"))
 
@@ -294,6 +329,24 @@ class VolumesAPIConfig(ConfigSectionInterface):
         """
         return json.loads(self.get('image_filter', '{}'))
 
+    @property
+    def image_filter_mode(self):
+        return self.get("image_filter_mode", 'inclusion')
+
+    @property
+    def flavor_filter(self):
+        """Expects Json.  Returns an empty dictionary by default (no filter).
+        Dictionary keys should be attributes of the flavor model, and key
+        values should be a list of values for that model attribute.
+        Used by some tests to decide which flavors to target for a given
+        test run.
+        """
+        return json.loads(self.get('flavor_filter', '{}'))
+
+    @property
+    def flavor_filter_mode(self):
+        return self.get("flavor_filter_mode", 'inclusion')
+
     @property
     def volume_type_filter(self):
         """Expects Json.  Returns an empty dictionary by default.
@@ -303,3 +356,13 @@ class VolumesAPIConfig(ConfigSectionInterface):
         test run.
         """
         return json.loads(self.get('volume_type_filter', '{}'))
+
+    @property
+    def volume_type_filter_mode(self):
+        return self.get("volume_type_filter_mode", 'inclusion')
+
+# API configuration
+    @property
+    def allow_snapshot_restore_to_different_type(self):
+        return self.get_boolean(
+            "allow_snapshot_restore_to_different_type", False)