openstack/openstacksdk
Code Issues Proposed changes

Support uploading image from data and stdin

Browse Source 
It might be useful to upload image from stdin. Disadvantage is, that it
is not possible to calculate checksums and this will prohibit threaded
image upload when using swift and tasks.
Additionally fix checksum validation during image creation - when
create_image is not called through cloud layer - the result might be
different, when checksums of existing image match (due to call and
return cloud.get_image). Fixing this requires also completing image v1
(proper find). Yeah, lots of tests are affected by that change.

Required-by: https://review.opendev.org/#/c/650374
Change-Id: I709d8b48cb7867fd806e2f19781bb84739363843
This commit is contained in:
Artem Goncharov 2019-12-13 18:24:18 +01:00
parent 3845dcc548
commit 63fe02bf77
8 changed files with 341 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
openstack/cloud/_object_store.py
View File

@ -219,14 +219,11 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
if file_key not in self._file_hash_cache:
self.log.debug(
'Calculating hashes for %(filename)s', {'filename': filename})
md5 = hashlib.md5()
sha256 = hashlib.sha256()
(md5, sha256) = (None, None)
with open(filename, 'rb') as file_obj:
for chunk in iter(lambda: file_obj.read(8192), b''):
md5.update(chunk)
sha256.update(chunk)
(md5, sha256) = self._calculate_data_hashes(file_obj)
self._file_hash_cache[file_key] = dict(
md5=md5.hexdigest(), sha256=sha256.hexdigest())
md5=md5, sha256=sha256)
self.log.debug(
"Image file %(filename)s md5:%(md5)s sha256:%(sha256)s",
{'filename': filename,
@ -235,6 +232,19 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
return (self._file_hash_cache[file_key]['md5'],
self._file_hash_cache[file_key]['sha256'])
def _calculate_data_hashes(self, data):
md5 = hashlib.md5()
sha256 = hashlib.sha256()
if hasattr(data, 'read'):
for chunk in iter(lambda: data.read(8192), b''):
md5.update(chunk)
sha256.update(chunk)
else:
md5.update(data)
sha256.update(data)
return (md5.hexdigest(), sha256.hexdigest())
@_utils.cache_on_arguments()
def get_object_capabilities(self):
"""Get infomation about the object-storage service

41
openstack/image/_base_proxy.py
View File

@ -14,6 +14,7 @@ import os
import six
from openstack import exceptions
from openstack import proxy
@ -40,7 +41,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
disable_vendor_agent=True,
allow_duplicates=False, meta=None,
wait=False, timeout=3600,
validate_checksum=True,
data=None, validate_checksum=True,
**kwargs):
"""Upload an image.
@ -49,6 +50,8 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
basename of the path.
:param str filename: The path to the file to upload, if needed.
(optional, defaults to None)
:param data: Image data (string or file-like object). It is mutually
exclusive with filename
:param str container: Name of the container in swift where images
should be uploaded for import if the cloud requires such a thing.
(optional, defaults to 'images')
@ -103,23 +106,34 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
# https://docs.openstack.org/image-guide/image-formats.html
container_format = 'bare'
if data and filename:
raise exceptions.SDKException(
'Passing filename and data simultaneously is not supported')
# If there is no filename, see if name is actually the filename
if not filename:
if not filename and not data:
name, filename = self._get_name_and_filename(
name, self._connection.config.config['image_format'])
if not (md5 or sha256):
if validate_checksum and data and not isinstance(data, bytes):
raise exceptions.SDKException(
'Validating checksum is not possible when data is not a '
'direct binary object')
if not (md5 or sha256) and validate_checksum:
if filename:
(md5, sha256) = self._connection._get_file_hashes(filename)
elif data and isinstance(data, bytes):
(md5, sha256) = self._connection._calculate_data_hashes(data)
if allow_duplicates:
current_image = None
else:
current_image = self._connection.get_image(name)
current_image = self.find_image(name)
if current_image:
md5_key = current_image.get(
props = current_image.get('properties', {})
md5_key = props.get(
self._IMAGE_MD5_KEY,
current_image.get(self._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = current_image.get(
props.get(self._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = props.get(
self._IMAGE_SHA256_KEY,
current_image.get(self._SHADE_IMAGE_SHA256_KEY, ''))
props.get(self._SHADE_IMAGE_SHA256_KEY, ''))
up_to_date = self._connection._hashes_up_to_date(
md5=md5, sha256=sha256,
md5_key=md5_key, sha256_key=sha256_key)
@ -128,6 +142,11 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
"image %(name)s exists and is up to date",
{'name': name})
return current_image
else:
self.log.debug(
"image %(name)s exists, but contains different "
"checksums. Updating.",
{'name': name})
if disable_vendor_agent:
kwargs.update(
@ -147,9 +166,9 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
if container_format:
image_kwargs['container_format'] = container_format
if filename:
if filename or data:
image = self._upload_image(
name, filename=filename, meta=meta,
name, filename=filename, data=data, meta=meta,
wait=wait, timeout=timeout,
validate_checksum=validate_checksum,
**image_kwargs)
@ -163,7 +182,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
pass
@abc.abstractmethod
def _upload_image(self, name, filename, meta, wait, timeout,
def _upload_image(self, name, filename, data, meta, wait, timeout,
validate_checksum=True,
**image_kwargs):
pass

5
openstack/image/v1/_proxy.py
View File

@ -42,10 +42,13 @@ class Proxy(_base_proxy.BaseImageProxy):
return self._create(_image.Image, **attrs)
def _upload_image(
self, name, filename, meta, wait, timeout, **image_kwargs):
self, name, filename, data, meta, wait, timeout, **image_kwargs):
# NOTE(mordred) wait and timeout parameters are unused, but
# are present for ease at calling site.
if filename and not data:
image_data = open(filename, 'rb')
else:
image_data = data
image_kwargs['properties'].update(meta)
image_kwargs['name'] = name

55
openstack/image/v1/image.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from openstack.image import _download
from openstack import exceptions
from openstack import resource
@ -29,6 +30,11 @@ class Image(resource.Resource, _download.DownloadMixin):
# Remotely they would be still in the resource root
_store_unknown_attrs_as_properties = True
_query_mapping = resource.QueryParameters(
'name', 'container_format', 'disk_format',
'status', 'size_min', 'size_max'
)
#: Hash of the image data used. The Image service uses this value
#: for verification.
checksum = resource.Body('checksum')
@ -73,3 +79,52 @@ class Image(resource.Resource, _download.DownloadMixin):
status = resource.Body('status')
#: The timestamp when this image was last updated.
updated_at = resource.Body('updated_at')
@classmethod
def find(cls, session, name_or_id, ignore_missing=True, **params):
"""Find a resource by its name or id.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param name_or_id: This resource's identifier, if needed by
the request. The default is ``None``.
:param bool ignore_missing: When set to ``False``
:class:`~openstack.exceptions.ResourceNotFound` will be
raised when the resource does not exist.
When set to ``True``, None will be returned when
attempting to find a nonexistent resource.
:param dict params: Any additional parameters to be passed into
underlying methods, such as to
:meth:`~openstack.resource.Resource.existing`
in order to pass on URI parameters.
:return: The :class:`Resource` object matching the given name or id
or None if nothing matches.
:raises: :class:`openstack.exceptions.DuplicateResource` if more
than one resource is found for this request.
:raises: :class:`openstack.exceptions.ResourceNotFound` if nothing
is found and ignore_missing is ``False``.
"""
session = cls._get_session(session)
# Try to short-circuit by looking directly for a matching ID.
try:
match = cls.existing(
id=name_or_id,
connection=session._get_connection(),
**params)
return match.fetch(session, **params)
except exceptions.NotFoundException:
pass
params['name'] = name_or_id
data = cls.list(session, base_path='/images/detail', **params)
result = cls._get_one_match(name_or_id, data)
if result is not None:
return result
if ignore_missing:
return None
raise exceptions.ResourceNotFound(
"No %s found for %s" % (cls.__name__, name_or_id))

15
openstack/image/v2/_proxy.py
View File

@ -148,7 +148,7 @@ class Proxy(_base_proxy.BaseImageProxy):
return img
def _upload_image(self, name, filename=None, meta=None,
def _upload_image(self, name, filename=None, data=None, meta=None,
wait=False, timeout=None, validate_checksum=True,
**kwargs):
# We can never have nice things. Glance v1 took "is_public" as a
@ -166,11 +166,11 @@ class Proxy(_base_proxy.BaseImageProxy):
# This makes me want to die inside
if self._connection.image_api_use_tasks:
return self._upload_image_task(
name, filename, meta=meta,
name, filename, data=data, meta=meta,
wait=wait, timeout=timeout, **kwargs)
else:
return self._upload_image_put(
name, filename, meta=meta,
name, filename, data=data, meta=meta,
validate_checksum=validate_checksum,
**kwargs)
except exceptions.SDKException:
@ -196,8 +196,12 @@ class Proxy(_base_proxy.BaseImageProxy):
return ret
def _upload_image_put(
self, name, filename, meta, validate_checksum, **image_kwargs):
self, name, filename, data, meta,
validate_checksum, **image_kwargs):
if filename and not data:
image_data = open(filename, 'rb')
else:
image_data = data
properties = image_kwargs.pop('properties', {})
@ -232,7 +236,7 @@ class Proxy(_base_proxy.BaseImageProxy):
return image
def _upload_image_task(
self, name, filename,
self, name, filename, data,
wait, timeout, meta, **image_kwargs):
if not self._connection.has_service('object-store'):
@ -251,6 +255,7 @@ class Proxy(_base_proxy.BaseImageProxy):
self._connection.create_object(
container, name, filename,
md5=md5, sha256=sha256,
data=data,
metadata={self._connection._OBJECT_AUTOCREATE_KEY: 'true'},
**{'content-type': 'application/octet-stream',
'x-delete-after': str(24 * 60 * 60)})

133
openstack/tests/unit/cloud/test_image.py
View File

@ -321,7 +321,21 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri=self.get_mock_url(
@ -356,6 +370,7 @@ class TestImage(BaseTestImage):
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
complete_qs=True,
json=self.fake_search_return)
])
@ -365,7 +380,7 @@ class TestImage(BaseTestImage):
is_public=False)
self.assert_calls()
self.assertEqual(self.adapter.request_history[5].text.read(),
self.assertEqual(self.adapter.request_history[7].text.read(),
self.output)
def test_create_image_task(self):
@ -390,7 +405,21 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='HEAD',
uri='{endpoint}/{container}'.format(
@ -517,6 +546,7 @@ class TestImage(BaseTestImage):
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
complete_qs=True,
json=self.fake_search_return)
])
@ -686,7 +716,11 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v1/images/detail',
uri='https://image.example.com/v1/images/' + self.image_name,
status_code=404),
dict(method='GET',
uri='https://image.example.com/v1/images/detail?name='
+ self.image_name,
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v1/images',
@ -726,7 +760,11 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v1/images/detail',
uri='https://image.example.com/v1/images/' + self.image_name,
status_code=404),
dict(method='GET',
uri='https://image.example.com/v1/images/detail?name='
+ self.image_name,
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v1/images',
@ -792,7 +830,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -828,10 +881,6 @@ class TestImage(BaseTestImage):
fake_image['owner_specified.openstack.sha256'] = 'b'
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
json={'images': []}),
dict(method='POST',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
@ -870,7 +919,8 @@ class TestImage(BaseTestImage):
exceptions.SDKException,
self.cloud.create_image,
self.image_name, self.imagefile.name,
is_public=False, md5='a', sha256='b'
is_public=False, md5='a', sha256='b',
allow_duplicates=True
)
self.assert_calls()
@ -878,15 +928,10 @@ class TestImage(BaseTestImage):
def test_create_image_put_bad_int(self):
self.cloud.image_api_use_tasks = False
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
json={'images': []}),
])
self.assertRaises(
exc.OpenStackCloudException,
self._call_create_image, self.image_name,
allow_duplicates=True,
min_disk='fish', min_ram=0)
self.assert_calls()
@ -910,7 +955,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -931,6 +991,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])
@ -959,7 +1020,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -980,6 +1056,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])
@ -1009,7 +1086,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -1030,6 +1122,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])

106
openstack/tests/unit/image/v2/test_proxy.py
View File

@ -11,6 +11,7 @@
# under the License.
import mock
import io
import requests
from openstack import exceptions
@ -41,6 +42,7 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
def setUp(self):
super(TestImageProxy, self).setUp()
self.proxy = _proxy.Proxy(self.session)
self.proxy._connection = self.cloud
def test_image_import_no_required_attrs(self):
# container_format and disk_format are required attrs of the image
@ -57,6 +59,110 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
expected_kwargs={"method": "method", "store": None,
"uri": "uri"})
def test_image_create_conflict(self):
self.assertRaises(
exceptions.SDKException, self.proxy.create_image,
name='fake', filename='fake', data='fake',
container='bare', disk_format='raw'
)
def test_image_create_checksum_match(self):
fake_image = image.Image(
id="fake", properties={
self.proxy._IMAGE_MD5_KEY: 'fake_md5',
self.proxy._IMAGE_SHA256_KEY: 'fake_sha256'
})
self.proxy.find_image = mock.Mock(return_value=fake_image)
self.proxy._upload_image = mock.Mock()
res = self.proxy.create_image(
name='fake',
md5='fake_md5', sha256='fake_sha256'
)
self.assertEqual(fake_image, res)
self.proxy._upload_image.assert_not_called()
def test_image_create_checksum_mismatch(self):
fake_image = image.Image(
id="fake", properties={
self.proxy._IMAGE_MD5_KEY: 'fake_md5',
self.proxy._IMAGE_SHA256_KEY: 'fake_sha256'
})
self.proxy.find_image = mock.Mock(return_value=fake_image)
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake',
md5='fake2_md5', sha256='fake2_sha256'
)
self.proxy._upload_image.assert_called()
def test_image_create_allow_duplicates_find_not_called(self):
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake', allow_duplicates=True,
)
self.proxy.find_image.assert_not_called()
def test_image_create_validate_checksum_data_binary(self):
""" Pass real data as binary"""
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake', validate_checksum=True,
container='bare', disk_format='raw'
)
self.proxy.find_image.assert_called_with('fake')
self.proxy._upload_image.assert_called_with(
'fake', container_format='bare', disk_format='raw',
filename=None, data=b'fake', meta={},
properties={
self.proxy._IMAGE_MD5_KEY: '144c9defac04969c7bfad8efaa8ea194',
self.proxy._IMAGE_SHA256_KEY: 'b5d54c39e66671c9731b9f471e585'
'd8262cd4f54963f0c93082d8dcf33'
'4d4c78',
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=True, wait=False)
def test_image_create_validate_checksum_data_not_binary(self):
self.assertRaises(
exceptions.SDKException, self.proxy.create_image,
name='fake', data=io.StringIO(), validate_checksum=True,
container='bare', disk_format='raw'
)
def test_image_create_data_binary(self):
"""Pass binary file-like object"""
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
data = io.BytesIO(b'\0\0')
self.proxy.create_image(
name='fake', data=data, validate_checksum=False,
container='bare', disk_format='raw'
)
self.proxy._upload_image.assert_called_with(
'fake', container_format='bare', disk_format='raw',
filename=None, data=data, meta={},
properties={
self.proxy._IMAGE_MD5_KEY: '',
self.proxy._IMAGE_SHA256_KEY: '',
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=False, wait=False)
def test_image_upload_no_args(self):
# container_format and disk_format are required args
self.assertRaises(exceptions.InvalidRequest, self.proxy.upload_image)

4
releasenotes/notes/support_stdin_image_upload-305c04fb2daeb32c.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Add support for creating image from STDIN (i.e. from OSC). When creating from STDIN however, no checksum verification is possible, and thus validate_checksum must be also set to False.