openstack/openstacksdk
Code Issues Proposed changes

Add support for multiple image stores

Browse Source 
Glance has support for having multiple backend stores. Add support
to image creation for specifying them.

Change-Id: I3b897abccfecf9353be07abc8f8325d91f3eb9d4
This commit is contained in:
Monty Taylor 2020-06-24 11:40:36 -05:00
parent 75ae5bf4aa
commit a66639f806
7 changed files with 225 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
openstack/image/_base_proxy.py
View File

@ -33,16 +33,20 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
_SHADE_IMAGE_OBJECT_KEY = 'owner_specified.shade.object'
def create_image(
self, name, filename=None,
container=None,
md5=None, sha256=None,
disk_format=None, container_format=None,
disable_vendor_agent=True,
allow_duplicates=False, meta=None,
wait=False, timeout=3600,
data=None, validate_checksum=False,
use_import=False,
**kwargs):
self, name, filename=None,
container=None,
md5=None, sha256=None,
disk_format=None, container_format=None,
disable_vendor_agent=True,
allow_duplicates=False, meta=None,
wait=False, timeout=3600,
data=None, validate_checksum=False,
use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
**kwargs,
):
"""Upload an image.
:param str name: Name of the image to create. If it is a pathname
@ -84,6 +88,26 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
the target cloud so should only be used when needed, such as when
the user needs the cloud to transform image format. If the cloud
has disabled direct uploads, this will default to true.
:param stores:
List of stores to be used when enabled_backends is activated
in glance. List values can be the id of a store or a
:class:`~openstack.image.v2.service_info.Store` instance.
Implies ``use_import`` equals ``True``.
:param all_stores:
Upload to all available stores. Mutually exclusive with
``store`` and ``stores``.
Implies ``use_import`` equals ``True``.
:param all_stores_must_succeed:
When set to True, if an error occurs during the upload in at
least one store, the worfklow fails, the data is deleted
from stores where copying is done (not staging), and the
state of the image is unchanged. When set to False, the
workflow will fail (data deleted from stores, …) only if the
import fails on all stores specified by the user. In case of
a partial success, the locations added to the image will be
the stores where the data has been correctly uploaded.
Default is True.
Implies ``use_import`` equals ``True``.
Additional kwargs will be passed to the image creation as additional
metadata for the image and will have all values converted to string
@ -177,6 +201,9 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
wait=wait, timeout=timeout,
validate_checksum=validate_checksum,
use_import=use_import,
stores=stores,
all_stores=stores,
all_stores_must_succeed=stores,
**image_kwargs)
else:
image_kwargs['name'] = name
@ -189,9 +216,14 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
pass
@abc.abstractmethod
def _upload_image(self, name, filename, data, meta, wait, timeout,
validate_checksum=True, use_import=False,
**image_kwargs):
def _upload_image(
self, name, filename, data, meta, wait, timeout,
validate_checksum=True, use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
**image_kwargs
):
pass
@abc.abstractmethod

8
openstack/image/v1/_proxy.py
View File

@ -45,11 +45,17 @@ class Proxy(_base_proxy.BaseImageProxy):
def _upload_image(
self, name, filename, data, meta, wait, timeout,
use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
**image_kwargs,
):
if use_import:
raise exceptions.SDKException(
raise exceptions.InvalidRequest(
"Glance v1 does not support image import")
if stores or all_stores or all_stores_must_succeed:
raise exceptions.InvalidRequest(
"Glance v1 does not support stores")
# NOTE(mordred) wait and timeout parameters are unused, but
# are present for ease at calling site.
if filename and not data:

93
openstack/image/v2/_proxy.py
View File

@ -36,8 +36,13 @@ class Proxy(_base_proxy.BaseImageProxy):
"""
return self._create(_image.Image, **kwargs)
def import_image(self, image, method='glance-direct', uri=None,
store=None):
def import_image(
self, image, method='glance-direct', uri=None,
store=None,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
):
"""Import data to an existing image
Interoperable image import process are introduced in the Image API
@ -45,24 +50,57 @@ class Proxy(_base_proxy.BaseImageProxy):
Image Service download it by itself without sending binary data at
image creation.
:param image: The value can be the ID of a image or a
:class:`~openstack.image.v2.image.Image` instance.
:param method: Method to use for importing the image.
A valid value is glance-direct or web-download.
:param uri: Required only if using the web-download import method.
This url is where the data is made available to the Image
service.
:param store: Used when enabled_backends is activated in glance
The value can be the id of a store or a
:class:`~openstack.image.v2.service_info.Store`
instance.
:param image:
The value can be the ID of a image or a
:class:`~openstack.image.v2.image.Image` instance.
:param method:
Method to use for importing the image.
A valid value is glance-direct or web-download.
:param uri:
Required only if using the web-download import method.
This url is where the data is made available to the Image
service.
:param store:
Used when enabled_backends is activated in glance. The value
can be the id of a store or a
:class:`~openstack.image.v2.service_info.Store` instance.
:param stores:
List of stores to be used when enabled_backends is activated
in glance. List values can be the id of a store or a
:class:`~openstack.image.v2.service_info.Store` instance.
:param all_stores:
Upload to all available stores. Mutually exclusive with
``store`` and ``stores``.
:param all_stores_must_succeed:
When set to True, if an error occurs during the upload in at
least one store, the worfklow fails, the data is deleted
from stores where copying is done (not staging), and the
state of the image is unchanged. When set to False, the
workflow will fail (data deleted from stores, …) only if the
import fails on all stores specified by the user. In case of
a partial success, the locations added to the image will be
the stores where the data has been correctly uploaded.
Default is True.
:returns: None
"""
image = self._get_resource(_image.Image, image)
if all_stores and (store or stores):
raise exceptions.InvalidRequest(
"all_stores is mutually exclusive with"
" store and stores")
if store is not None:
if stores:
raise exceptions.InvalidRequest(
"store and stores are mutually exclusive")
store = self._get_resource(_si.Store, store)
stores = stores or []
new_stores = []
for s in stores:
new_stores.append(self._get_resource(_si.Store, s))
stores = new_stores
# as for the standard image upload function, container_format and
# disk_format are required for using image import process
if not all([image.container_format, image.disk_format]):
@ -70,7 +108,13 @@ class Proxy(_base_proxy.BaseImageProxy):
"Both container_format and disk_format are required for"
" importing an image")
image.import_image(self, method=method, uri=uri, store=store)
image.import_image(
self, method=method, uri=uri,
store=store,
stores=stores,
all_stores=all_stores,
all_stores_must_succeed=all_stores_must_succeed,
)
def stage_image(self, image, filename=None, data=None):
"""Stage binary image data
@ -148,10 +192,15 @@ class Proxy(_base_proxy.BaseImageProxy):
return img
def _upload_image(self, name, filename=None, data=None, meta=None,
wait=False, timeout=None, validate_checksum=True,
use_import=False,
**kwargs):
def _upload_image(
self, name, filename=None, data=None, meta=None,
wait=False, timeout=None, validate_checksum=True,
use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
**kwargs
):
# We can never have nice things. Glance v1 took "is_public" as a
# boolean. Glance v2 takes "visibility". If the user gives us
# is_public, we know what they mean. If they give us visibility, they
@ -180,6 +229,9 @@ class Proxy(_base_proxy.BaseImageProxy):
name, filename, data=data, meta=meta,
validate_checksum=validate_checksum,
use_import=use_import,
stores=stores,
all_stores=all_stores,
all_stores_must_succeed=all_stores_must_succeed,
**kwargs)
except exceptions.SDKException:
self.log.debug("Image creation failed", exc_info=True)
@ -206,6 +258,9 @@ class Proxy(_base_proxy.BaseImageProxy):
def _upload_image_put(
self, name, filename, data, meta,
validate_checksum, use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
**image_kwargs,
):
if filename and not data:
@ -225,6 +280,8 @@ class Proxy(_base_proxy.BaseImageProxy):
image.image_import_methods
and 'glance-direct' in image.image_import_methods
)
if stores or all_stores or all_stores_must_succeed:
use_import = True
if use_import and not supports_import:
raise exceptions.SDKException(
"Importing image was requested but the cloud does not"

25
openstack/image/v2/image.py
View File

@ -269,8 +269,22 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
return self
def import_image(self, session, method='glance-direct', uri=None,
store=None):
store=None, stores=None, all_stores=None,
all_stores_must_succeed=None):
"""Import Image via interoperable image import process"""
if all_stores and (store or stores):
raise exceptions.InvalidRequest(
"all_stores is mutually exclusive with"
" store and stores")
if store and stores:
raise exceptions.InvalidRequest(
"store and stores are mutually exclusive."
" Please just use stores.")
if store:
stores = [store]
else:
stores = stores or []
url = utils.urljoin(self.base_path, self.id, 'import')
json = {'method': {'name': method}}
if uri:
@ -279,7 +293,16 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
else:
raise exceptions.InvalidRequest('URI is only supported with '
'method: "web-download"')
if all_stores is not None:
json['all_stores'] = all_stores
if all_stores_must_succeed is not None:
json['all_stores_must_succeed'] = all_stores_must_succeed
for s in stores:
json.setdefault('stores', [])
json['stores'].append(s.id)
headers = {}
# Backward compat
if store is not None:
headers = {'X-Image-Meta-Store': store.id}
session.post(url, json=json, headers=headers)

54
openstack/tests/unit/image/v2/test_image.py
View File

@ -277,9 +277,15 @@ class TestImage(base.TestCase):
json={"method": {"name": "glance-direct"}}
)
def test_import_image_with_stores(self):
def test_import_image_with_store(self):
sot = image.Image(**EXAMPLE)
json = {"method": {"name": "web-download", "uri": "such-a-good-uri"}}
json = {
"method": {
"name": "web-download",
"uri": "such-a-good-uri",
},
"stores": ["ceph_1"],
}
store = mock.MagicMock()
store.id = "ceph_1"
sot.import_image(self.sess, "web-download", "such-a-good-uri", store)
@ -289,6 +295,50 @@ class TestImage(base.TestCase):
json=json
)
def test_import_image_with_stores(self):
sot = image.Image(**EXAMPLE)
json = {
"method": {
"name": "web-download",
"uri": "such-a-good-uri",
},
"stores": ["ceph_1"],
}
store = mock.MagicMock()
store.id = "ceph_1"
sot.import_image(
self.sess,
"web-download",
"such-a-good-uri",
stores=[store],
)
self.sess.post.assert_called_with(
'images/IDENTIFIER/import',
headers={},
json=json,
)
def test_import_image_with_all_stores(self):
sot = image.Image(**EXAMPLE)
json = {
"method": {
"name": "web-download",
"uri": "such-a-good-uri",
},
"all_stores": True,
}
sot.import_image(
self.sess,
"web-download",
"such-a-good-uri",
all_stores=True,
)
self.sess.post.assert_called_with(
'images/IDENTIFIER/import',
headers={},
json=json,
)
def test_upload(self):
sot = image.Image(**EXAMPLE)

23
openstack/tests/unit/image/v2/test_proxy.py
View File

@ -53,11 +53,18 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
def test_image_import(self):
original_image = image.Image(**EXAMPLE)
self._verify("openstack.image.v2.image.Image.import_image",
self.proxy.import_image,
method_args=[original_image, "method", "uri"],
expected_kwargs={"method": "method", "store": None,
"uri": "uri"})
self._verify(
"openstack.image.v2.image.Image.import_image",
self.proxy.import_image,
method_args=[original_image, "method", "uri"],
expected_kwargs={
"method": "method",
"store": None,
"uri": "uri",
"stores": [],
"all_stores": None,
"all_stores_must_succeed": None,
})
def test_image_create_conflict(self):
self.assertRaises(
@ -134,6 +141,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=True,
use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
wait=False)
def test_image_create_validate_checksum_data_not_binary(self):
@ -165,6 +175,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=False,
use_import=False,
stores=None,
all_stores=None,
all_stores_must_succeed=None,
wait=False)
def test_image_create_without_filename(self):

4
releasenotes/notes/glance-image-stores-2baa66e6743a2f2d.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Add support for specifying stores when doing glance image uploads.