Make glance v2 the default

This adds some compatibility to ensure that both Glance wrappers
support both is_public and visibility kwargs.

Co-Authored-By: Andrey Kurilin <andr.kurilin@gmail.com>

Change-Id: If27fffe83b7f610c802fcb29c7442801ac6ca908

rally-jobs/cinder.yaml

@@ -123,7 +123,7 @@
           images_per_tenant: 1
           image_name: "image-context-test"
           image_args:
-            is_public: True
+            visibility: "public"
       sla:
         failure_rate:
           max: 0

rally-jobs/nova.yaml

@@ -680,7 +680,7 @@
           images_per_tenant: 1
           image_name: "rally-named-image-from-context"
           image_args:
-            is_public: True
+            visibility: "public"
       sla:
         failure_rate:
           max: 0

rally-jobs/rally.yaml

@@ -880,6 +880,9 @@
         users:
           tenants: 2
           users_per_tenant: 3
+        api_versions:
+          glance:
+            version: 1
       sla:
         failure_rate:
           max: 0

rally/osclients.py

@@ -360,7 +360,7 @@ class Neutron(OSClient):
         return client
 
 
-@configure("glance", default_version="1", default_service_type="image",
+@configure("glance", default_version="2", default_service_type="image",
            supported_versions=["1", "2"])
 class Glance(OSClient):
     def create_client(self, version=None, service_type=None):

rally/plugins/openstack/context/glance/images.py

@@ -102,6 +102,10 @@ class ImageGenerator(context.Context):
                 LOG.warning("The 'min_disk' argument is deprecated; specify "
                             "arbitrary arguments with 'image_args' instead")
                 kwargs["min_disk"] = self.config["min_disk"]
+            if "is_public" in kwargs:
+                LOG.warning("The 'is_public' argument is deprecated since "
+                            "Rally 0.8.0; specify visibility arguments "
+                            "instead")
 
             for i in range(images_per_tenant):
                 if image_name and i > 0:

rally/plugins/openstack/scenarios/glance/images.py

@@ -26,7 +26,8 @@ LOG = logging.getLogger(__name__)
 """Scenarios for Glance images."""
 
 
-@types.convert(image_location={"type": "path_or_url"})
+@types.convert(image_location={"type": "path_or_url"},
+               kwargs={"type": "glance_image_args"})
 @validation.required_services(consts.Service.GLANCE)
 @validation.required_openstack(users=True)
 @scenario.configure(context={"cleanup": ["glance"]},
@@ -79,6 +80,8 @@ class ListImages(utils.GlanceScenario, nova_utils.NovaScenario):
         self._list_images()
 
 
+@types.convert(image_location={"type": "path_or_url"},
+               kwargs={"type": "glance_image_args"})
 @validation.required_services(consts.Service.GLANCE)
 @validation.required_openstack(users=True)
 @scenario.configure(context={"cleanup": ["glance"]},
@@ -102,7 +105,9 @@ class CreateAndDeleteImage(utils.GlanceScenario, nova_utils.NovaScenario):
         self._delete_image(image)
 
 
-@types.convert(flavor={"type": "nova_flavor"})
+@types.convert(flavor={"type": "nova_flavor"},
+               image_location={"type": "path_or_url"},
+               kwargs={"type": "glance_image_args"})
 @validation.flavor_exists("flavor")
 @validation.required_services(consts.Service.GLANCE, consts.Service.NOVA)
 @validation.required_openstack(users=True)

rally/plugins/openstack/types.py

@@ -12,6 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
+
 from rally.common.plugin import plugin
 from rally import exceptions
 from rally.task import types
@@ -87,6 +89,29 @@ class GlanceImage(types.ResourceType):
         return resource_id
 
 
+@plugin.configure(name="glance_image_args")
+class GlanceImageArguments(types.ResourceType):
+
+    @classmethod
+    def transform(cls, clients, resource_config):
+        """Transform the resource config to id.
+
+        :param clients: openstack admin client handles
+        :param resource_config: scenario config with `id`, `name` or `regex`
+
+        :returns: id matching resource
+        """
+        resource_config = copy.deepcopy(resource_config)
+        if "is_public" in resource_config:
+            if "visibility" in resource_config:
+                resource_config.pop("is_public")
+            else:
+                visibility = ("public" if resource_config.pop("is_public")
+                              else "private")
+                resource_config["visibility"] = visibility
+        return resource_config
+
+
 @plugin.configure(name="ec2_image")
 class EC2Image(types.ResourceType):
 
@@ -213,4 +238,4 @@ class WatcherGoal(types.ResourceType):
                     resource_config.get("name"))],
                 typename="goal",
                 id_attr="uuid")
-        return resource_id
+        return resource_id

rally/plugins/openstack/verification/tempest/config.py

@@ -116,41 +116,6 @@ def _create_or_get_data_dir():
     return data_dir
 
 
-def _download_image(image_path, image=None):
-    if image:
-        LOG.debug("Downloading image '%s' "
-                  "from Glance to %s" % (image.name, image_path))
-        with open(image_path, "wb") as image_file:
-            for chunk in image.data():
-                image_file.write(chunk)
-    else:
-        LOG.debug("Downloading image from %s "
-                  "to %s" % (CONF.tempest.img_url, image_path))
-        try:
-            response = requests.get(CONF.tempest.img_url, stream=True)
-        except requests.ConnectionError as err:
-            msg = _("Failed to download image. "
-                    "Possibly there is no connection to Internet. "
-                    "Error: %s.") % (str(err) or "unknown")
-            raise exceptions.TempestConfigCreationFailure(msg)
-
-        if response.status_code == 200:
-            with open(image_path, "wb") as image_file:
-                for chunk in response.iter_content(chunk_size=1024):
-                    if chunk:   # filter out keep-alive new chunks
-                        image_file.write(chunk)
-                        image_file.flush()
-        else:
-            if response.status_code == 404:
-                msg = _("Failed to download image. Image was not found.")
-            else:
-                msg = _("Failed to download image. "
-                        "HTTP error code %d.") % response.status_code
-            raise exceptions.TempestConfigCreationFailure(msg)
-
-    LOG.debug("The image has been successfully downloaded!")
-
-
 def write_configfile(path, conf_object):
     with open(path, "w") as configfile:
         conf_object.write(configfile)
@@ -448,6 +413,40 @@ class TempestResourcesContext(context.VerifierContext):
         LOG.debug("There is no public image with name matching "
                   "regular expression '%s'" % CONF.tempest.img_name_regex)
 
+    def _do_download_image(self, image_path, image=None):
+        if image:
+            LOG.debug("Downloading image '%s' "
+                      "from Glance to %s" % (image.name, image_path))
+            with open(image_path, "wb") as image_file:
+                for chunk in self.clients.glance().images.data(image.id):
+                    image_file.write(chunk)
+        else:
+            LOG.debug("Downloading image from %s "
+                      "to %s" % (CONF.tempest.img_url, image_path))
+            try:
+                response = requests.get(CONF.tempest.img_url, stream=True)
+            except requests.ConnectionError as err:
+                msg = _("Failed to download image. "
+                        "Possibly there is no connection to Internet. "
+                        "Error: %s.") % (str(err) or "unknown")
+                raise exceptions.TempestConfigCreationFailure(msg)
+
+            if response.status_code == 200:
+                with open(image_path, "wb") as image_file:
+                    for chunk in response.iter_content(chunk_size=1024):
+                        if chunk:   # filter out keep-alive new chunks
+                            image_file.write(chunk)
+                            image_file.flush()
+            else:
+                if response.status_code == 404:
+                    msg = _("Failed to download image. Image was not found.")
+                else:
+                    msg = _("Failed to download image. "
+                            "HTTP error code %d.") % response.status_code
+                raise exceptions.TempestConfigCreationFailure(msg)
+
+        LOG.debug("The image has been successfully downloaded!")
+
     def _download_image(self):
         image_path = os.path.join(self.data_dir, self.image_name)
         if os.path.isfile(image_path):
@@ -457,9 +456,9 @@ class TempestResourcesContext(context.VerifierContext):
         if CONF.tempest.img_name_regex:
             image = self._discover_image()
             if image:
-                return _download_image(image_path, image)
+                return self._do_download_image(image_path, image)
 
-        _download_image(image_path)
+        self._do_download_image(image_path)
 
     def _configure_option(self, section, option, value=None,
                           helper_method=None, *args, **kwargs):

rally/plugins/openstack/wrappers/glance.py

@@ -158,6 +158,10 @@ class GlanceV2Wrapper(GlanceWrapper):
         kw.update(kwargs)
         if "name" not in kw:
             kw["name"] = self.owner.generate_random_name()
+        if "is_public" in kw:
+            LOG.warning("is_public is not supported by Glance v2, and is "
+                        "deprecated in Rally v0.8.0")
+            kw["visibility"] = "public" if kw.pop("is_public") else "private"
 
         image_location = os.path.expanduser(image_location)
 

rally/task/validation.py

@@ -265,7 +265,12 @@ def _get_validated_image(config, clients, param_name):
     try:
         image_id = openstack_types.GlanceImage.transform(
             clients=clients, resource_config=image_args)
-        image = clients.glance().images.get(image=image_id).to_dict()
+        image = clients.glance().images.get(image_id)
+        if hasattr(image, "to_dict"):
+            # NOTE(stpierre): Glance v1 images are objects that can be
+            # converted to dicts; Glance v2 images are already
+            # dict-like
+            image = image.to_dict()
         if not image.get("size"):
             image["size"] = 0
         if not image.get("min_ram"):

tests/unit/plugins/openstack/test_types.py

@@ -177,6 +177,23 @@ class GlanceImageTestCase(test.TestCase):
                           resource_config)
 
 
+class GlanceImageArgsTestCase(test.TestCase):
+
+    def test_transform(self):
+        self.assertEqual({}, types.GlanceImageArguments.transform(
+            clients=None, resource_config={}))
+        self.assertEqual(
+            {"visibility": "public"}, types.GlanceImageArguments.transform(
+                clients=None, resource_config={"visibility": "public"}))
+        self.assertEqual(
+            {"visibility": "public"}, types.GlanceImageArguments.transform(
+                clients=None, resource_config={"visibility": "public",
+                                               "is_public": False}))
+        self.assertEqual(
+            {"visibility": "private"}, types.GlanceImageArguments.transform(
+                clients=None, resource_config={"is_public": False}))
+
+
 class EC2ImageTestCase(test.TestCase):
 
     def setUp(self):

tests/unit/plugins/openstack/verification/tempest/test_config.py

@@ -270,10 +270,12 @@ class TempestResourcesContextTestCase(test.TestCase):
         self.mock_isfile.return_value = False
         img_path = os.path.join(self.context.data_dir, "foo")
         img = mock.MagicMock()
-        img.data.return_value = "data"
+        glanceclient = self.context.clients.glance()
+        glanceclient.images.data.return_value = "data"
 
-        config._download_image(img_path, img)
+        self.context._do_download_image(img_path, img)
         mock_open.assert_called_once_with(img_path, "wb")
+        glanceclient.images.data.assert_called_once_with(img.id)
         mock_open().write.assert_has_calls([mock.call("d"),
                                             mock.call("a"),
                                             mock.call("t"),
@@ -286,7 +288,7 @@ class TempestResourcesContextTestCase(test.TestCase):
         img_path = os.path.join(self.context.data_dir, "foo")
         mock_get.return_value.iter_content.return_value = "data"
 
-        config._download_image(img_path)
+        self.context._do_download_image(img_path)
         mock_get.assert_called_once_with(CONF.tempest.img_url, stream=True)
         mock_open.assert_called_once_with(img_path, "wb")
         mock_open().write.assert_has_calls([mock.call("d"),
@@ -300,7 +302,8 @@ class TempestResourcesContextTestCase(test.TestCase):
         self.mock_isfile.return_value = False
         mock_get.return_value = mock.MagicMock(status_code=status_code)
         self.assertRaises(
-            exceptions.TempestConfigCreationFailure, config._download_image,
+            exceptions.TempestConfigCreationFailure,
+            self.context._do_download_image,
             os.path.join(self.context.data_dir, "foo"))
 
     @mock.patch("requests.get", side_effect=requests.ConnectionError())
@@ -308,7 +311,8 @@ class TempestResourcesContextTestCase(test.TestCase):
             self, mock_requests_get):
         self.mock_isfile.return_value = False
         self.assertRaises(
-            exceptions.TempestConfigCreationFailure, config._download_image,
+            exceptions.TempestConfigCreationFailure,
+            self.context._do_download_image,
             os.path.join(self.context.data_dir, "foo"))
 
     @mock.patch("rally.plugins.openstack.wrappers."
@@ -372,11 +376,15 @@ class TempestResourcesContextTestCase(test.TestCase):
         img_1.name = "Foo"
         img_2 = mock.MagicMock()
         img_2.name = "CirrOS"
-        img_2.data.return_value = "data"
+        glanceclient = self.context.clients.glance()
+        glanceclient.images.data.return_value = "data"
         mock_wrap.return_value.list_images.return_value = [img_1, img_2]
 
         self.context._download_image()
         img_path = os.path.join(self.context.data_dir, self.context.image_name)
+        mock_wrap.return_value.list_images.assert_called_once_with(
+            status="active", visibility="public")
+        glanceclient.images.data.assert_called_once_with(img_2.id)
         mock_open.assert_called_once_with(img_path, "wb")
         mock_open().write.assert_has_calls([mock.call("d"),
                                             mock.call("a"),

tests/unit/plugins/openstack/wrappers/test_glance.py

@@ -197,12 +197,14 @@ class GlanceV2WrapperTestCase(test.ScenarioTestCase):
         {"location": "image_location", "visibility": "private"},
         {"location": "image_location", "fakearg": "fake"},
         {"location": "image_location", "name": "image_name"},
-        {"location": _tempfile.name, "visibility": "public"})
+        {"location": _tempfile.name, "visibility": "public"},
+        {"location": "image_location",
+         "expected_kwargs": {"visibility": "public"}, "is_public": True})
     @ddt.unpack
     @mock.patch("six.moves.builtins.open")
     @mock.patch("requests.get")
     def test_create_image(self, mock_requests_get, mock_open, location,
-                          **kwargs):
+                          expected_kwargs=None, **kwargs):
         self.wrapped_client.get_image = mock.Mock()
         created_image = mock.Mock()
         uploaded_image = mock.Mock()
@@ -213,11 +215,11 @@ class GlanceV2WrapperTestCase(test.ScenarioTestCase):
                                                         location,
                                                         "disk_format",
                                                         **kwargs)
-        create_args = kwargs
+        create_args = expected_kwargs or kwargs
         create_args["container_format"] = "container_format"
         create_args["disk_format"] = "disk_format"
-        if "name" not in kwargs:
-            create_args["name"] = self.owner.generate_random_name.return_value
+        create_args.setdefault("name",
+                               self.owner.generate_random_name.return_value)
 
         self.client().images.create.assert_called_once_with(**create_args)
 

tests/unit/task/test_validation.py

@@ -268,7 +268,7 @@ class ValidatorsTestCase(test.TestCase):
                          result[1])
         mock_glance_image_transform.assert_called_once_with(
             clients=clients, resource_config="test")
-        clients.glance().images.get.assert_called_with(image="image_id")
+        clients.glance().images.get.assert_called_with("image_id")
 
     @mock.patch(MODULE + "openstack_types.GlanceImage.transform",
                 side_effect=exceptions.InvalidScenarioArgument)

tests/unit/test_osclients.py

@@ -383,7 +383,7 @@ class OSClientsTestCase(test.TestCase):
             client = self.clients.glance()
             self.assertEqual(fake_glance, client)
             kw = {
-                "version": "1",
+                "version": "2",
                 "session": mock_keystoneauth1.session.Session(),
                 "endpoint_override": mock_glance__get_endpoint.return_value}
             mock_glance.Client.assert_called_once_with(**kw)