Add support for multiple image stores

Glance has support for having multiple backend stores. Add support
to image creation for specifying them.

Change-Id: I3b897abccfecf9353be07abc8f8325d91f3eb9d4

openstack/image/_base_proxy.py

@@ -33,16 +33,20 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
     _SHADE_IMAGE_OBJECT_KEY = 'owner_specified.shade.object'
 
     def create_image(
-            self, name, filename=None,
+        self, name, filename=None,
-            container=None,
+        container=None,
-            md5=None, sha256=None,
+        md5=None, sha256=None,
-            disk_format=None, container_format=None,
+        disk_format=None, container_format=None,
-            disable_vendor_agent=True,
+        disable_vendor_agent=True,
-            allow_duplicates=False, meta=None,
+        allow_duplicates=False, meta=None,
-            wait=False, timeout=3600,
+        wait=False, timeout=3600,
-            data=None, validate_checksum=False,
+        data=None, validate_checksum=False,
-            use_import=False,
+        use_import=False,
-            **kwargs):
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
+        **kwargs,
+    ):
         """Upload an image.
 
         :param str name: Name of the image to create. If it is a pathname
@@ -84,6 +88,26 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
             the target cloud so should only be used when needed, such as when
             the user needs the cloud to transform image format. If the cloud
             has disabled direct uploads, this will default to true.
+        :param stores:
+            List of stores to be used when enabled_backends is activated
+            in glance. List values can be the id of a store or a
+            :class:`~openstack.image.v2.service_info.Store` instance.
+            Implies ``use_import`` equals ``True``.
+        :param all_stores:
+            Upload to all available stores. Mutually exclusive with
+            ``store`` and ``stores``.
+            Implies ``use_import`` equals ``True``.
+        :param all_stores_must_succeed:
+            When set to True, if an error occurs during the upload in at
+            least one store, the worfklow fails, the data is deleted
+            from stores where copying is done (not staging), and the
+            state of the image is unchanged. When set to False, the
+            workflow will fail (data deleted from stores, …) only if the
+            import fails on all stores specified by the user. In case of
+            a partial success, the locations added to the image will be
+            the stores where the data has been correctly uploaded.
+            Default is True.
+            Implies ``use_import`` equals ``True``.
 
         Additional kwargs will be passed to the image creation as additional
         metadata for the image and will have all values converted to string
@@ -177,6 +201,9 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
                 wait=wait, timeout=timeout,
                 validate_checksum=validate_checksum,
                 use_import=use_import,
+                stores=stores,
+                all_stores=stores,
+                all_stores_must_succeed=stores,
                 **image_kwargs)
         else:
             image_kwargs['name'] = name
@@ -189,9 +216,14 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
         pass
 
     @abc.abstractmethod
-    def _upload_image(self, name, filename, data, meta, wait, timeout,
+    def _upload_image(
-                      validate_checksum=True, use_import=False,
+        self, name, filename, data, meta, wait, timeout,
-                      **image_kwargs):
+        validate_checksum=True, use_import=False,
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
+        **image_kwargs
+    ):
         pass
 
     @abc.abstractmethod

openstack/image/v1/_proxy.py

@@ -45,11 +45,17 @@ class Proxy(_base_proxy.BaseImageProxy):
     def _upload_image(
         self, name, filename, data, meta, wait, timeout,
         use_import=False,
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
         **image_kwargs,
     ):
         if use_import:
-            raise exceptions.SDKException(
+            raise exceptions.InvalidRequest(
                 "Glance v1 does not support image import")
+        if stores or all_stores or all_stores_must_succeed:
+            raise exceptions.InvalidRequest(
+                "Glance v1 does not support stores")
         # NOTE(mordred) wait and timeout parameters are unused, but
         # are present for ease at calling site.
         if filename and not data:

openstack/image/v2/_proxy.py

@@ -36,8 +36,13 @@ class Proxy(_base_proxy.BaseImageProxy):
         """
         return self._create(_image.Image, **kwargs)
 
-    def import_image(self, image, method='glance-direct', uri=None,
+    def import_image(
-                     store=None):
+        self, image, method='glance-direct', uri=None,
+        store=None,
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
+    ):
         """Import data to an existing image
 
         Interoperable image import process are introduced in the Image API
@@ -45,24 +50,57 @@ class Proxy(_base_proxy.BaseImageProxy):
         Image Service download it by itself without sending binary data at
         image creation.
 
-        :param image: The value can be the ID of a image or a
+        :param image:
-                      :class:`~openstack.image.v2.image.Image` instance.
+            The value can be the ID of a image or a
-        :param method: Method to use for importing the image.
+            :class:`~openstack.image.v2.image.Image` instance.
-                       A valid value is glance-direct or web-download.
+        :param method:
-        :param uri: Required only if using the web-download import method.
+            Method to use for importing the image.
-                    This url is where the data is made available to the Image
+            A valid value is glance-direct or web-download.
-                    service.
+        :param uri:
-        :param store: Used when enabled_backends is activated in glance
+            Required only if using the web-download import method.
-                      The value can be the id of a store or a
+            This url is where the data is made available to the Image
-                      :class:`~openstack.image.v2.service_info.Store`
+            service.
-                      instance.
+        :param store:
+            Used when enabled_backends is activated in glance. The value
+            can be the id of a store or a
+            :class:`~openstack.image.v2.service_info.Store` instance.
+        :param stores:
+            List of stores to be used when enabled_backends is activated
+            in glance. List values can be the id of a store or a
+            :class:`~openstack.image.v2.service_info.Store` instance.
+        :param all_stores:
+            Upload to all available stores. Mutually exclusive with
+            ``store`` and ``stores``.
+        :param all_stores_must_succeed:
+            When set to True, if an error occurs during the upload in at
+            least one store, the worfklow fails, the data is deleted
+            from stores where copying is done (not staging), and the
+            state of the image is unchanged. When set to False, the
+            workflow will fail (data deleted from stores, …) only if the
+            import fails on all stores specified by the user. In case of
+            a partial success, the locations added to the image will be
+            the stores where the data has been correctly uploaded.
+            Default is True.
 
         :returns: None
         """
         image = self._get_resource(_image.Image, image)
+        if all_stores and (store or stores):
+            raise exceptions.InvalidRequest(
+                "all_stores is mutually exclusive with"
+                " store and stores")
         if store is not None:
+            if stores:
+                raise exceptions.InvalidRequest(
+                    "store and stores are mutually exclusive")
             store = self._get_resource(_si.Store, store)
 
+        stores = stores or []
+        new_stores = []
+        for s in stores:
+            new_stores.append(self._get_resource(_si.Store, s))
+        stores = new_stores
+
         # as for the standard image upload function, container_format and
         # disk_format are required for using image import process
         if not all([image.container_format, image.disk_format]):
@@ -70,7 +108,13 @@ class Proxy(_base_proxy.BaseImageProxy):
                 "Both container_format and disk_format are required for"
                 " importing an image")
 
-        image.import_image(self, method=method, uri=uri, store=store)
+        image.import_image(
+            self, method=method, uri=uri,
+            store=store,
+            stores=stores,
+            all_stores=all_stores,
+            all_stores_must_succeed=all_stores_must_succeed,
+        )
 
     def stage_image(self, image, filename=None, data=None):
         """Stage binary image data
@@ -148,10 +192,15 @@ class Proxy(_base_proxy.BaseImageProxy):
 
         return img
 
-    def _upload_image(self, name, filename=None, data=None, meta=None,
+    def _upload_image(
-                      wait=False, timeout=None, validate_checksum=True,
+        self, name, filename=None, data=None, meta=None,
-                      use_import=False,
+        wait=False, timeout=None, validate_checksum=True,
-                      **kwargs):
+        use_import=False,
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
+        **kwargs
+    ):
         # We can never have nice things. Glance v1 took "is_public" as a
         # boolean. Glance v2 takes "visibility". If the user gives us
         # is_public, we know what they mean. If they give us visibility, they
@@ -180,6 +229,9 @@ class Proxy(_base_proxy.BaseImageProxy):
                     name, filename, data=data, meta=meta,
                     validate_checksum=validate_checksum,
                     use_import=use_import,
+                    stores=stores,
+                    all_stores=all_stores,
+                    all_stores_must_succeed=all_stores_must_succeed,
                     **kwargs)
         except exceptions.SDKException:
             self.log.debug("Image creation failed", exc_info=True)
@@ -206,6 +258,9 @@ class Proxy(_base_proxy.BaseImageProxy):
     def _upload_image_put(
         self, name, filename, data, meta,
         validate_checksum, use_import=False,
+        stores=None,
+        all_stores=None,
+        all_stores_must_succeed=None,
         **image_kwargs,
     ):
         if filename and not data:
@@ -225,6 +280,8 @@ class Proxy(_base_proxy.BaseImageProxy):
             image.image_import_methods
             and 'glance-direct' in image.image_import_methods
         )
+        if stores or all_stores or all_stores_must_succeed:
+            use_import = True
         if use_import and not supports_import:
             raise exceptions.SDKException(
                 "Importing image was requested but the cloud does not"

openstack/image/v2/image.py

@@ -269,8 +269,22 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
         return self
 
     def import_image(self, session, method='glance-direct', uri=None,
-                     store=None):
+                     store=None, stores=None, all_stores=None,
+                     all_stores_must_succeed=None):
         """Import Image via interoperable image import process"""
+        if all_stores and (store or stores):
+            raise exceptions.InvalidRequest(
+                "all_stores is mutually exclusive with"
+                " store and stores")
+        if store and stores:
+            raise exceptions.InvalidRequest(
+                "store and stores are mutually exclusive."
+                " Please just use stores.")
+        if store:
+            stores = [store]
+        else:
+            stores = stores or []
+
         url = utils.urljoin(self.base_path, self.id, 'import')
         json = {'method': {'name': method}}
         if uri:
@@ -279,7 +293,16 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
             else:
                 raise exceptions.InvalidRequest('URI is only supported with '
                                                 'method: "web-download"')
+        if all_stores is not None:
+            json['all_stores'] = all_stores
+        if all_stores_must_succeed is not None:
+            json['all_stores_must_succeed'] = all_stores_must_succeed
+        for s in stores:
+            json.setdefault('stores', [])
+            json['stores'].append(s.id)
+
         headers = {}
+        # Backward compat
         if store is not None:
             headers = {'X-Image-Meta-Store': store.id}
         session.post(url, json=json, headers=headers)

openstack/tests/unit/image/v2/test_image.py

@@ -277,9 +277,15 @@ class TestImage(base.TestCase):
             json={"method": {"name": "glance-direct"}}
         )
 
-    def test_import_image_with_stores(self):
+    def test_import_image_with_store(self):
         sot = image.Image(**EXAMPLE)
-        json = {"method": {"name": "web-download", "uri": "such-a-good-uri"}}
+        json = {
+            "method": {
+                "name": "web-download",
+                "uri": "such-a-good-uri",
+            },
+            "stores": ["ceph_1"],
+        }
         store = mock.MagicMock()
         store.id = "ceph_1"
         sot.import_image(self.sess, "web-download", "such-a-good-uri", store)
@@ -289,6 +295,50 @@ class TestImage(base.TestCase):
             json=json
         )
 
+    def test_import_image_with_stores(self):
+        sot = image.Image(**EXAMPLE)
+        json = {
+            "method": {
+                "name": "web-download",
+                "uri": "such-a-good-uri",
+            },
+            "stores": ["ceph_1"],
+        }
+        store = mock.MagicMock()
+        store.id = "ceph_1"
+        sot.import_image(
+            self.sess,
+            "web-download",
+            "such-a-good-uri",
+            stores=[store],
+        )
+        self.sess.post.assert_called_with(
+            'images/IDENTIFIER/import',
+            headers={},
+            json=json,
+        )
+
+    def test_import_image_with_all_stores(self):
+        sot = image.Image(**EXAMPLE)
+        json = {
+            "method": {
+                "name": "web-download",
+                "uri": "such-a-good-uri",
+            },
+            "all_stores": True,
+        }
+        sot.import_image(
+            self.sess,
+            "web-download",
+            "such-a-good-uri",
+            all_stores=True,
+        )
+        self.sess.post.assert_called_with(
+            'images/IDENTIFIER/import',
+            headers={},
+            json=json,
+        )
+
     def test_upload(self):
         sot = image.Image(**EXAMPLE)
 

openstack/tests/unit/image/v2/test_proxy.py

@@ -53,11 +53,18 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
 
     def test_image_import(self):
         original_image = image.Image(**EXAMPLE)
-        self._verify("openstack.image.v2.image.Image.import_image",
+        self._verify(
-                     self.proxy.import_image,
+            "openstack.image.v2.image.Image.import_image",
-                     method_args=[original_image, "method", "uri"],
+            self.proxy.import_image,
-                     expected_kwargs={"method": "method", "store": None,
+            method_args=[original_image, "method", "uri"],
-                                      "uri": "uri"})
+            expected_kwargs={
+                "method": "method",
+                "store": None,
+                "uri": "uri",
+                "stores": [],
+                "all_stores": None,
+                "all_stores_must_succeed": None,
+            })
 
     def test_image_create_conflict(self):
         self.assertRaises(
@@ -134,6 +141,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
                 self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
             timeout=3600, validate_checksum=True,
             use_import=False,
+            stores=None,
+            all_stores=None,
+            all_stores_must_succeed=None,
             wait=False)
 
     def test_image_create_validate_checksum_data_not_binary(self):
@@ -165,6 +175,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
                 self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
             timeout=3600, validate_checksum=False,
             use_import=False,
+            stores=None,
+            all_stores=None,
+            all_stores_must_succeed=None,
             wait=False)
 
     def test_image_create_without_filename(self):

releasenotes/notes/glance-image-stores-2baa66e6743a2f2d.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add support for specifying stores when doing glance image uploads.