675 lines
25 KiB
Python
675 lines
25 KiB
Python
# Copyright 2014 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from collections import abc
|
|
import copy
|
|
import functools
|
|
|
|
from cryptography import exceptions as crypto_exception
|
|
from cursive import exception as cursive_exception
|
|
from cursive import signature_utils
|
|
import glance_store as store
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
from oslo_utils import encodeutils
|
|
from oslo_utils import excutils
|
|
|
|
from glance.common import exception
|
|
from glance.common import format_inspector
|
|
from glance.common import store_utils
|
|
from glance.common import utils
|
|
import glance.domain.proxy
|
|
from glance.i18n import _, _LE, _LI, _LW
|
|
|
|
|
|
CONF = cfg.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class ImageRepoProxy(glance.domain.proxy.Repo):
|
|
|
|
def __init__(self, image_repo, context, store_api, store_utils):
|
|
self.context = context
|
|
self.store_api = store_api
|
|
self.image_repo = image_repo
|
|
proxy_kwargs = {'context': context, 'store_api': store_api,
|
|
'store_utils': store_utils}
|
|
super(ImageRepoProxy, self).__init__(image_repo,
|
|
item_proxy_class=ImageProxy,
|
|
item_proxy_kwargs=proxy_kwargs)
|
|
|
|
self.db_api = glance.db.get_api()
|
|
|
|
def _set_acls(self, image):
|
|
public = image.visibility in ['public', 'community']
|
|
member_ids = []
|
|
if image.locations and not public:
|
|
member_repo = _get_member_repo_for_store(image,
|
|
self.context,
|
|
self.db_api,
|
|
self.store_api)
|
|
member_ids = [m.member_id for m in member_repo.list()]
|
|
for location in image.locations:
|
|
if CONF.enabled_backends:
|
|
self.store_api.set_acls_for_multi_store(
|
|
location['url'], location['metadata'].get('store'),
|
|
public=public, read_tenants=member_ids,
|
|
context=self.context
|
|
)
|
|
else:
|
|
self.store_api.set_acls(location['url'], public=public,
|
|
read_tenants=member_ids,
|
|
context=self.context)
|
|
|
|
def add(self, image):
|
|
result = super(ImageRepoProxy, self).add(image)
|
|
self._set_acls(image)
|
|
return result
|
|
|
|
def save(self, image, from_state=None):
|
|
result = super(ImageRepoProxy, self).save(image, from_state=from_state)
|
|
self._set_acls(image)
|
|
return result
|
|
|
|
def get(self, image_id):
|
|
image = super(ImageRepoProxy, self).get(image_id)
|
|
if CONF.enabled_backends:
|
|
try:
|
|
store_utils.update_store_in_locations(
|
|
self.context, image, self.image_repo)
|
|
except exception.Forbidden:
|
|
# NOTE(danms): We may not be able to complete a store
|
|
# update if we do not own the image. That should not
|
|
# break us, so avoid raising Forbidden in that
|
|
# case. Note that modifications to @image here will
|
|
# still be returned to the user, just not saved in the
|
|
# DB. That is probably what we want anyway.
|
|
pass
|
|
return image
|
|
|
|
|
|
def _get_member_repo_for_store(image, context, db_api, store_api):
|
|
image_member_repo = glance.db.ImageMemberRepo(context, db_api, image)
|
|
store_image_repo = glance.location.ImageMemberRepoProxy(
|
|
image_member_repo, image, context, store_api)
|
|
|
|
return store_image_repo
|
|
|
|
|
|
def _check_location_uri(context, store_api, store_utils, uri,
|
|
backend=None):
|
|
"""Check if an image location is valid.
|
|
|
|
:param context: Glance request context
|
|
:param store_api: store API module
|
|
:param store_utils: store utils module
|
|
:param uri: location's uri string
|
|
:param backend: A backend name for the store
|
|
"""
|
|
|
|
try:
|
|
# NOTE(zhiyan): Some stores return zero when it catch exception
|
|
if CONF.enabled_backends:
|
|
size_from_backend = store_api.get_size_from_uri_and_backend(
|
|
uri, backend, context=context)
|
|
else:
|
|
size_from_backend = store_api.get_size_from_backend(
|
|
uri, context=context)
|
|
|
|
is_ok = (store_utils.validate_external_location(uri) and
|
|
size_from_backend > 0)
|
|
except (store.UnknownScheme, store.NotFound, store.BadStoreUri):
|
|
is_ok = False
|
|
if not is_ok:
|
|
reason = _('Invalid location')
|
|
raise exception.BadStoreUri(message=reason)
|
|
|
|
|
|
def _check_image_location(context, store_api, store_utils, location):
|
|
backend = None
|
|
if CONF.enabled_backends:
|
|
backend = location['metadata'].get('store')
|
|
|
|
_check_location_uri(context, store_api, store_utils, location['url'],
|
|
backend=backend)
|
|
store_api.check_location_metadata(location['metadata'])
|
|
|
|
|
|
def _set_image_size(context, image, locations):
|
|
if not image.size:
|
|
for location in locations:
|
|
if CONF.enabled_backends:
|
|
size_from_backend = store.get_size_from_uri_and_backend(
|
|
location['url'], location['metadata'].get('store'),
|
|
context=context)
|
|
else:
|
|
size_from_backend = store.get_size_from_backend(
|
|
location['url'], context=context)
|
|
|
|
if size_from_backend:
|
|
# NOTE(flwang): This assumes all locations have the same size
|
|
image.size = size_from_backend
|
|
break
|
|
|
|
|
|
def _count_duplicated_locations(locations, new):
|
|
"""
|
|
To calculate the count of duplicated locations for new one.
|
|
|
|
:param locations: The exiting image location set
|
|
:param new: The new image location
|
|
:returns: The count of duplicated locations
|
|
"""
|
|
|
|
ret = 0
|
|
for loc in locations:
|
|
if loc['url'] == new['url'] and loc['metadata'] == new['metadata']:
|
|
ret += 1
|
|
return ret
|
|
|
|
|
|
class ImageFactoryProxy(glance.domain.proxy.ImageFactory):
|
|
def __init__(self, factory, context, store_api, store_utils):
|
|
self.context = context
|
|
self.store_api = store_api
|
|
self.store_utils = store_utils
|
|
proxy_kwargs = {'context': context, 'store_api': store_api,
|
|
'store_utils': store_utils}
|
|
super(ImageFactoryProxy, self).__init__(factory,
|
|
proxy_class=ImageProxy,
|
|
proxy_kwargs=proxy_kwargs)
|
|
|
|
def new_image(self, **kwargs):
|
|
locations = kwargs.get('locations', [])
|
|
for loc in locations:
|
|
_check_image_location(self.context,
|
|
self.store_api,
|
|
self.store_utils,
|
|
loc)
|
|
loc['status'] = 'active'
|
|
if _count_duplicated_locations(locations, loc) > 1:
|
|
raise exception.DuplicateLocation(location=loc['url'])
|
|
return super(ImageFactoryProxy, self).new_image(**kwargs)
|
|
|
|
|
|
@functools.total_ordering
|
|
class StoreLocations(abc.MutableSequence):
|
|
"""
|
|
The proxy for store location property. It takes responsibility for::
|
|
|
|
1. Location uri correctness checking when adding a new location.
|
|
2. Remove the image data from the store when a location is removed
|
|
from an image.
|
|
|
|
"""
|
|
def __init__(self, image_proxy, value):
|
|
self.image_proxy = image_proxy
|
|
if isinstance(value, list):
|
|
self.value = value
|
|
else:
|
|
self.value = list(value)
|
|
|
|
def append(self, location):
|
|
# NOTE(flaper87): Insert this
|
|
# location at the very end of
|
|
# the value list.
|
|
self.insert(len(self.value), location)
|
|
|
|
def extend(self, other):
|
|
if isinstance(other, StoreLocations):
|
|
locations = other.value
|
|
else:
|
|
locations = list(other)
|
|
|
|
for location in locations:
|
|
self.append(location)
|
|
|
|
def insert(self, i, location):
|
|
_check_image_location(self.image_proxy.context,
|
|
self.image_proxy.store_api,
|
|
self.image_proxy.store_utils,
|
|
location)
|
|
location['status'] = 'active'
|
|
if _count_duplicated_locations(self.value, location) > 0:
|
|
raise exception.DuplicateLocation(location=location['url'])
|
|
|
|
self.value.insert(i, location)
|
|
_set_image_size(self.image_proxy.context,
|
|
self.image_proxy,
|
|
[location])
|
|
|
|
def pop(self, i=-1):
|
|
location = self.value.pop(i)
|
|
try:
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
except store.exceptions.NotFound:
|
|
# NOTE(rosmaita): This can happen if the data was deleted by an
|
|
# operator from the backend, or a race condition from multiple
|
|
# delete-from-store requests. The old way to deal with this was
|
|
# that the user could just delete the image when the data is gone,
|
|
# but with multi-store, that is no longer a good option. So we
|
|
# intentionally leave the location popped (in other words, the
|
|
# pop() succeeds) but we also reraise the NotFound so that the
|
|
# calling code knows what happened.
|
|
with excutils.save_and_reraise_exception():
|
|
pass
|
|
except Exception:
|
|
with excutils.save_and_reraise_exception():
|
|
self.value.insert(i, location)
|
|
return location
|
|
|
|
def count(self, location):
|
|
return self.value.count(location)
|
|
|
|
def index(self, location, *args):
|
|
return self.value.index(location, *args)
|
|
|
|
def remove(self, location):
|
|
if self.count(location):
|
|
self.pop(self.index(location))
|
|
else:
|
|
self.value.remove(location)
|
|
|
|
def reverse(self):
|
|
self.value.reverse()
|
|
|
|
# Mutable sequence, so not hashable
|
|
__hash__ = None
|
|
|
|
def __getitem__(self, i):
|
|
return self.value.__getitem__(i)
|
|
|
|
def __setitem__(self, i, location):
|
|
_check_image_location(self.image_proxy.context,
|
|
self.image_proxy.store_api,
|
|
self.image_proxy.store_utils,
|
|
location)
|
|
location['status'] = 'active'
|
|
self.value.__setitem__(i, location)
|
|
_set_image_size(self.image_proxy.context,
|
|
self.image_proxy,
|
|
[location])
|
|
|
|
def __delitem__(self, i):
|
|
if isinstance(i, slice):
|
|
if i.step not in (None, 1):
|
|
raise NotImplementedError("slice with step")
|
|
self.__delslice__(i.start, i.stop)
|
|
return
|
|
location = None
|
|
try:
|
|
location = self.value[i]
|
|
except Exception:
|
|
del self.value[i]
|
|
return
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
del self.value[i]
|
|
|
|
def __delslice__(self, i, j):
|
|
i = 0 if i is None else max(i, 0)
|
|
j = len(self) if j is None else max(j, 0)
|
|
locations = []
|
|
try:
|
|
locations = self.value[i:j]
|
|
except Exception:
|
|
del self.value[i:j]
|
|
return
|
|
for location in locations:
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
del self.value[i]
|
|
|
|
def __iadd__(self, other):
|
|
self.extend(other)
|
|
return self
|
|
|
|
def __contains__(self, location):
|
|
return location in self.value
|
|
|
|
def __len__(self):
|
|
return len(self.value)
|
|
|
|
def __cast(self, other):
|
|
if isinstance(other, StoreLocations):
|
|
return other.value
|
|
else:
|
|
return other
|
|
|
|
def __eq__(self, other):
|
|
return self.value == self.__cast(other)
|
|
|
|
def __lt__(self, other):
|
|
return self.value < self.__cast(other)
|
|
|
|
def __iter__(self):
|
|
return iter(self.value)
|
|
|
|
def __copy__(self):
|
|
return type(self)(self.image_proxy, self.value)
|
|
|
|
def __deepcopy__(self, memo):
|
|
# NOTE(zhiyan): Only copy location entries, others can be reused.
|
|
value = copy.deepcopy(self.value, memo)
|
|
self.image_proxy.image.locations = value
|
|
return type(self)(self.image_proxy, value)
|
|
|
|
|
|
def _locations_proxy(target, attr):
|
|
"""
|
|
Make a location property proxy on the image object.
|
|
|
|
:param target: the image object on which to add the proxy
|
|
:param attr: the property proxy we want to hook
|
|
"""
|
|
def get_attr(self):
|
|
value = getattr(getattr(self, target), attr)
|
|
return StoreLocations(self, value)
|
|
|
|
def set_attr(self, value):
|
|
if not isinstance(value, (list, StoreLocations)):
|
|
reason = _('Invalid locations')
|
|
raise exception.BadStoreUri(message=reason)
|
|
ori_value = getattr(getattr(self, target), attr)
|
|
if ori_value != value:
|
|
# NOTE(flwang): If all the URL of passed-in locations are same as
|
|
# current image locations, that means user would like to only
|
|
# update the metadata, not the URL.
|
|
ordered_value = sorted([loc['url'] for loc in value])
|
|
ordered_ori = sorted([loc['url'] for loc in ori_value])
|
|
if len(ori_value) > 0 and ordered_value != ordered_ori:
|
|
raise exception.Invalid(_('Original locations is not empty: '
|
|
'%s') % ori_value)
|
|
# NOTE(zhiyan): Check locations are all valid
|
|
# NOTE(flwang): If all the URL of passed-in locations are same as
|
|
# current image locations, then it's not necessary to verify those
|
|
# locations again. Otherwise, if there is any restricted scheme in
|
|
# existing locations. _check_image_location will fail.
|
|
if ordered_value != ordered_ori:
|
|
for loc in value:
|
|
_check_image_location(self.context,
|
|
self.store_api,
|
|
self.store_utils,
|
|
loc)
|
|
loc['status'] = 'active'
|
|
if _count_duplicated_locations(value, loc) > 1:
|
|
raise exception.DuplicateLocation(location=loc['url'])
|
|
_set_image_size(self.context, getattr(self, target), value)
|
|
else:
|
|
for loc in value:
|
|
loc['status'] = 'active'
|
|
return setattr(getattr(self, target), attr, list(value))
|
|
|
|
def del_attr(self):
|
|
value = getattr(getattr(self, target), attr)
|
|
while len(value):
|
|
self.store_utils.delete_image_location_from_backend(
|
|
self.context,
|
|
self.image.image_id,
|
|
value[0])
|
|
del value[0]
|
|
setattr(getattr(self, target), attr, value)
|
|
return delattr(getattr(self, target), attr)
|
|
|
|
return property(get_attr, set_attr, del_attr)
|
|
|
|
|
|
class ImageProxy(glance.domain.proxy.Image):
|
|
|
|
locations = _locations_proxy('image', 'locations')
|
|
|
|
def __init__(self, image, context, store_api, store_utils):
|
|
self.image = image
|
|
self.context = context
|
|
self.store_api = store_api
|
|
self.store_utils = store_utils
|
|
proxy_kwargs = {
|
|
'context': context,
|
|
'image': self,
|
|
'store_api': store_api,
|
|
}
|
|
super(ImageProxy, self).__init__(
|
|
image, member_repo_proxy_class=ImageMemberRepoProxy,
|
|
member_repo_proxy_kwargs=proxy_kwargs)
|
|
|
|
def delete(self):
|
|
self.image.delete()
|
|
if self.image.locations:
|
|
for location in self.image.locations:
|
|
self.store_utils.delete_image_location_from_backend(
|
|
self.context,
|
|
self.image.image_id,
|
|
location)
|
|
|
|
def _upload_to_store(self, data, verifier, store=None, size=None):
|
|
"""
|
|
Upload data to store
|
|
|
|
:param data: data to upload to store
|
|
:param verifier: for signature verification
|
|
:param store: store to upload data to
|
|
:param size: data size
|
|
:return:
|
|
"""
|
|
hashing_algo = self.image.os_hash_algo or CONF['hashing_algorithm']
|
|
if CONF.enabled_backends:
|
|
(location, size, checksum,
|
|
multihash, loc_meta) = self.store_api.add_with_multihash(
|
|
CONF,
|
|
self.image.image_id,
|
|
utils.LimitingReader(utils.CooperativeReader(data),
|
|
CONF.image_size_cap),
|
|
size,
|
|
store,
|
|
hashing_algo,
|
|
context=self.context,
|
|
verifier=verifier)
|
|
else:
|
|
(location,
|
|
size,
|
|
checksum,
|
|
multihash,
|
|
loc_meta) = self.store_api.add_to_backend_with_multihash(
|
|
CONF,
|
|
self.image.image_id,
|
|
utils.LimitingReader(utils.CooperativeReader(data),
|
|
CONF.image_size_cap),
|
|
size,
|
|
hashing_algo,
|
|
context=self.context,
|
|
verifier=verifier)
|
|
self._verify_signature(verifier, location, loc_meta)
|
|
for attr, data in {"size": size, "os_hash_value": multihash,
|
|
"checksum": checksum}.items():
|
|
self._verify_uploaded_data(data, attr)
|
|
self.image.locations.append({'url': location, 'metadata': loc_meta,
|
|
'status': 'active'})
|
|
self.image.checksum = checksum
|
|
self.image.os_hash_value = multihash
|
|
self.image.size = size
|
|
self.image.os_hash_algo = hashing_algo
|
|
|
|
def _verify_signature(self, verifier, location, loc_meta):
|
|
"""
|
|
Verify signature of uploaded data.
|
|
|
|
:param verifier: for signature verification
|
|
"""
|
|
# NOTE(bpoulos): if verification fails, exception will be raised
|
|
if verifier is not None:
|
|
try:
|
|
verifier.verify()
|
|
msg = _LI("Successfully verified signature for image %s")
|
|
LOG.info(msg, self.image.image_id)
|
|
except crypto_exception.InvalidSignature:
|
|
if CONF.enabled_backends:
|
|
self.store_api.delete(location,
|
|
loc_meta.get('store'),
|
|
context=self.context)
|
|
else:
|
|
self.store_api.delete_from_backend(location,
|
|
context=self.context)
|
|
raise cursive_exception.SignatureVerificationError(
|
|
_('Signature verification failed')
|
|
)
|
|
|
|
def _verify_uploaded_data(self, value, attribute_name):
|
|
"""
|
|
Verify value of attribute_name uploaded data
|
|
|
|
:param value: value to compare
|
|
:param attribute_name: attribute name of the image to compare with
|
|
"""
|
|
image_value = getattr(self.image, attribute_name)
|
|
if image_value is not None and value != image_value:
|
|
msg = _("%s of uploaded data is different from current "
|
|
"value set on the image.")
|
|
LOG.error(msg, attribute_name)
|
|
raise exception.UploadException(msg % attribute_name)
|
|
|
|
def set_data(self, data, size=None, backend=None, set_active=True):
|
|
if size is None:
|
|
size = 0 # NOTE(markwash): zero -> unknown size
|
|
|
|
# Create the verifier for signature verification (if correct properties
|
|
# are present)
|
|
extra_props = self.image.extra_properties
|
|
verifier = None
|
|
if signature_utils.should_create_verifier(extra_props):
|
|
# NOTE(bpoulos): if creating verifier fails, exception will be
|
|
# raised
|
|
img_signature = extra_props[signature_utils.SIGNATURE]
|
|
hash_method = extra_props[signature_utils.HASH_METHOD]
|
|
key_type = extra_props[signature_utils.KEY_TYPE]
|
|
cert_uuid = extra_props[signature_utils.CERT_UUID]
|
|
verifier = signature_utils.get_verifier(
|
|
context=self.context,
|
|
img_signature_certificate_uuid=cert_uuid,
|
|
img_signature_hash_method=hash_method,
|
|
img_signature=img_signature,
|
|
img_signature_key_type=key_type
|
|
)
|
|
|
|
if not self.image.virtual_size:
|
|
inspector = format_inspector.get_inspector(self.image.disk_format)
|
|
else:
|
|
# No need to do this again
|
|
inspector = None
|
|
|
|
if inspector and self.image.container_format == 'bare':
|
|
fmt = inspector()
|
|
data = format_inspector.InfoWrapper(data, fmt)
|
|
LOG.debug('Enabling in-flight format inspection for %s', fmt)
|
|
else:
|
|
fmt = None
|
|
|
|
self._upload_to_store(data, verifier, backend, size)
|
|
|
|
virtual_size = 0
|
|
if fmt and fmt.format_match:
|
|
try:
|
|
virtual_size = fmt.virtual_size
|
|
LOG.info('Image format matched and virtual size computed: %i',
|
|
virtual_size)
|
|
except Exception as e:
|
|
LOG.error(_LE('Unable to determine virtual_size because: %s'),
|
|
e)
|
|
elif fmt:
|
|
LOG.warning('Image format %s did not match; '
|
|
'unable to calculate virtual size',
|
|
self.image.disk_format)
|
|
|
|
if virtual_size:
|
|
self.image.virtual_size = fmt.virtual_size
|
|
|
|
if set_active and self.image.status != 'active':
|
|
self.image.status = 'active'
|
|
|
|
def get_data(self, offset=0, chunk_size=None):
|
|
if not self.image.locations:
|
|
# NOTE(mclaren): This is the only set of arguments
|
|
# which work with this exception currently, see:
|
|
# https://bugs.launchpad.net/glance-store/+bug/1501443
|
|
# When the above glance_store bug is fixed we can
|
|
# add a msg as usual.
|
|
raise store.NotFound(image=None)
|
|
err = None
|
|
for loc in self.image.locations:
|
|
try:
|
|
backend = loc['metadata'].get('store')
|
|
if CONF.enabled_backends:
|
|
data, size = self.store_api.get(
|
|
loc['url'], backend, offset=offset,
|
|
chunk_size=chunk_size, context=self.context
|
|
)
|
|
else:
|
|
data, size = self.store_api.get_from_backend(
|
|
loc['url'],
|
|
offset=offset,
|
|
chunk_size=chunk_size,
|
|
context=self.context)
|
|
|
|
return data
|
|
except Exception as e:
|
|
LOG.warning(_LW('Get image %(id)s data failed: '
|
|
'%(err)s.'),
|
|
{'id': self.image.image_id,
|
|
'err': encodeutils.exception_to_unicode(e)})
|
|
err = e
|
|
# tried all locations
|
|
LOG.error(_LE('Glance tried all active locations to get data for '
|
|
'image %s but all have failed.'), self.image.image_id)
|
|
raise err
|
|
|
|
|
|
class ImageMemberRepoProxy(glance.domain.proxy.Repo):
|
|
def __init__(self, repo, image, context, store_api):
|
|
self.repo = repo
|
|
self.image = image
|
|
self.context = context
|
|
self.store_api = store_api
|
|
super(ImageMemberRepoProxy, self).__init__(repo)
|
|
|
|
def _set_acls(self):
|
|
public = self.image.visibility in ['public', 'community']
|
|
if self.image.locations and not public:
|
|
member_ids = [m.member_id for m in self.repo.list()]
|
|
for location in self.image.locations:
|
|
if CONF.enabled_backends:
|
|
self.store_api.set_acls_for_multi_store(
|
|
location['url'], location['metadata'].get('store'),
|
|
public=public, read_tenants=member_ids,
|
|
context=self.context
|
|
)
|
|
else:
|
|
self.store_api.set_acls(location['url'], public=public,
|
|
read_tenants=member_ids,
|
|
context=self.context)
|
|
|
|
def add(self, member):
|
|
super(ImageMemberRepoProxy, self).add(member)
|
|
self._set_acls()
|
|
|
|
def remove(self, member):
|
|
super(ImageMemberRepoProxy, self).remove(member)
|
|
self._set_acls()
|