f20daa18e4
We added compatibility handling in I11e482354725ed74629dcaa9f2b7f229a60aa2a8 for the move of some classes from collections to collections.abc and incompatibility between older and newer python versions. Now that we have dropped the older versions, all supported versions have the classes under collections.abc and we can drop the special handling. Change-Id: I6d7cb25700ed552fbf82c09b6eea688ebee0dab2 Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>
625 lines
23 KiB
Python
625 lines
23 KiB
Python
# Copyright 2014 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from collections import abc
|
|
import copy
|
|
import functools
|
|
|
|
from cryptography import exceptions as crypto_exception
|
|
from cursive import exception as cursive_exception
|
|
from cursive import signature_utils
|
|
import glance_store as store
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
from oslo_utils import encodeutils
|
|
from oslo_utils import excutils
|
|
|
|
from glance.common import exception
|
|
from glance.common import utils
|
|
import glance.domain.proxy
|
|
from glance.i18n import _, _LE, _LI, _LW
|
|
|
|
|
|
CONF = cfg.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class ImageRepoProxy(glance.domain.proxy.Repo):
|
|
|
|
def __init__(self, image_repo, context, store_api, store_utils):
|
|
self.context = context
|
|
self.store_api = store_api
|
|
proxy_kwargs = {'context': context, 'store_api': store_api,
|
|
'store_utils': store_utils}
|
|
super(ImageRepoProxy, self).__init__(image_repo,
|
|
item_proxy_class=ImageProxy,
|
|
item_proxy_kwargs=proxy_kwargs)
|
|
|
|
self.db_api = glance.db.get_api()
|
|
|
|
def _set_acls(self, image):
|
|
public = image.visibility == 'public'
|
|
member_ids = []
|
|
if image.locations and not public:
|
|
member_repo = _get_member_repo_for_store(image,
|
|
self.context,
|
|
self.db_api,
|
|
self.store_api)
|
|
member_ids = [m.member_id for m in member_repo.list()]
|
|
for location in image.locations:
|
|
if CONF.enabled_backends:
|
|
self.store_api.set_acls_for_multi_store(
|
|
location['url'], location['metadata'].get('store'),
|
|
public=public, read_tenants=member_ids,
|
|
context=self.context
|
|
)
|
|
else:
|
|
self.store_api.set_acls(location['url'], public=public,
|
|
read_tenants=member_ids,
|
|
context=self.context)
|
|
|
|
def add(self, image):
|
|
result = super(ImageRepoProxy, self).add(image)
|
|
self._set_acls(image)
|
|
return result
|
|
|
|
def save(self, image, from_state=None):
|
|
result = super(ImageRepoProxy, self).save(image, from_state=from_state)
|
|
self._set_acls(image)
|
|
return result
|
|
|
|
|
|
def _get_member_repo_for_store(image, context, db_api, store_api):
|
|
image_member_repo = glance.db.ImageMemberRepo(context, db_api, image)
|
|
store_image_repo = glance.location.ImageMemberRepoProxy(
|
|
image_member_repo, image, context, store_api)
|
|
|
|
return store_image_repo
|
|
|
|
|
|
def _check_location_uri(context, store_api, store_utils, uri,
|
|
backend=None):
|
|
"""Check if an image location is valid.
|
|
|
|
:param context: Glance request context
|
|
:param store_api: store API module
|
|
:param store_utils: store utils module
|
|
:param uri: location's uri string
|
|
:param backend: A backend name for the store
|
|
"""
|
|
|
|
try:
|
|
# NOTE(zhiyan): Some stores return zero when it catch exception
|
|
if CONF.enabled_backends:
|
|
size_from_backend = store_api.get_size_from_uri_and_backend(
|
|
uri, backend, context=context)
|
|
else:
|
|
size_from_backend = store_api.get_size_from_backend(
|
|
uri, context=context)
|
|
|
|
is_ok = (store_utils.validate_external_location(uri) and
|
|
size_from_backend > 0)
|
|
except (store.UnknownScheme, store.NotFound, store.BadStoreUri):
|
|
is_ok = False
|
|
if not is_ok:
|
|
reason = _('Invalid location')
|
|
raise exception.BadStoreUri(message=reason)
|
|
|
|
|
|
def _check_image_location(context, store_api, store_utils, location):
|
|
backend = None
|
|
if CONF.enabled_backends:
|
|
backend = location['metadata'].get('store')
|
|
|
|
_check_location_uri(context, store_api, store_utils, location['url'],
|
|
backend=backend)
|
|
store_api.check_location_metadata(location['metadata'])
|
|
|
|
|
|
def _set_image_size(context, image, locations):
|
|
if not image.size:
|
|
for location in locations:
|
|
if CONF.enabled_backends:
|
|
size_from_backend = store.get_size_from_uri_and_backend(
|
|
location['url'], location['metadata'].get('store'),
|
|
context=context)
|
|
else:
|
|
size_from_backend = store.get_size_from_backend(
|
|
location['url'], context=context)
|
|
|
|
if size_from_backend:
|
|
# NOTE(flwang): This assumes all locations have the same size
|
|
image.size = size_from_backend
|
|
break
|
|
|
|
|
|
def _count_duplicated_locations(locations, new):
|
|
"""
|
|
To calculate the count of duplicated locations for new one.
|
|
|
|
:param locations: The exiting image location set
|
|
:param new: The new image location
|
|
:returns: The count of duplicated locations
|
|
"""
|
|
|
|
ret = 0
|
|
for loc in locations:
|
|
if loc['url'] == new['url'] and loc['metadata'] == new['metadata']:
|
|
ret += 1
|
|
return ret
|
|
|
|
|
|
class ImageFactoryProxy(glance.domain.proxy.ImageFactory):
|
|
def __init__(self, factory, context, store_api, store_utils):
|
|
self.context = context
|
|
self.store_api = store_api
|
|
self.store_utils = store_utils
|
|
proxy_kwargs = {'context': context, 'store_api': store_api,
|
|
'store_utils': store_utils}
|
|
super(ImageFactoryProxy, self).__init__(factory,
|
|
proxy_class=ImageProxy,
|
|
proxy_kwargs=proxy_kwargs)
|
|
|
|
def new_image(self, **kwargs):
|
|
locations = kwargs.get('locations', [])
|
|
for loc in locations:
|
|
_check_image_location(self.context,
|
|
self.store_api,
|
|
self.store_utils,
|
|
loc)
|
|
loc['status'] = 'active'
|
|
if _count_duplicated_locations(locations, loc) > 1:
|
|
raise exception.DuplicateLocation(location=loc['url'])
|
|
return super(ImageFactoryProxy, self).new_image(**kwargs)
|
|
|
|
|
|
@functools.total_ordering
|
|
class StoreLocations(abc.MutableSequence):
|
|
"""
|
|
The proxy for store location property. It takes responsibility for::
|
|
|
|
1. Location uri correctness checking when adding a new location.
|
|
2. Remove the image data from the store when a location is removed
|
|
from an image.
|
|
|
|
"""
|
|
def __init__(self, image_proxy, value):
|
|
self.image_proxy = image_proxy
|
|
if isinstance(value, list):
|
|
self.value = value
|
|
else:
|
|
self.value = list(value)
|
|
|
|
def append(self, location):
|
|
# NOTE(flaper87): Insert this
|
|
# location at the very end of
|
|
# the value list.
|
|
self.insert(len(self.value), location)
|
|
|
|
def extend(self, other):
|
|
if isinstance(other, StoreLocations):
|
|
locations = other.value
|
|
else:
|
|
locations = list(other)
|
|
|
|
for location in locations:
|
|
self.append(location)
|
|
|
|
def insert(self, i, location):
|
|
_check_image_location(self.image_proxy.context,
|
|
self.image_proxy.store_api,
|
|
self.image_proxy.store_utils,
|
|
location)
|
|
location['status'] = 'active'
|
|
if _count_duplicated_locations(self.value, location) > 0:
|
|
raise exception.DuplicateLocation(location=location['url'])
|
|
|
|
self.value.insert(i, location)
|
|
_set_image_size(self.image_proxy.context,
|
|
self.image_proxy,
|
|
[location])
|
|
|
|
def pop(self, i=-1):
|
|
location = self.value.pop(i)
|
|
try:
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
except store.exceptions.NotFound:
|
|
# NOTE(rosmaita): This can happen if the data was deleted by an
|
|
# operator from the backend, or a race condition from multiple
|
|
# delete-from-store requests. The old way to deal with this was
|
|
# that the user could just delete the image when the data is gone,
|
|
# but with multi-store, that is no longer a good option. So we
|
|
# intentionally leave the location popped (in other words, the
|
|
# pop() succeeds) but we also reraise the NotFound so that the
|
|
# calling code knows what happened.
|
|
with excutils.save_and_reraise_exception():
|
|
pass
|
|
except Exception:
|
|
with excutils.save_and_reraise_exception():
|
|
self.value.insert(i, location)
|
|
return location
|
|
|
|
def count(self, location):
|
|
return self.value.count(location)
|
|
|
|
def index(self, location, *args):
|
|
return self.value.index(location, *args)
|
|
|
|
def remove(self, location):
|
|
if self.count(location):
|
|
self.pop(self.index(location))
|
|
else:
|
|
self.value.remove(location)
|
|
|
|
def reverse(self):
|
|
self.value.reverse()
|
|
|
|
# Mutable sequence, so not hashable
|
|
__hash__ = None
|
|
|
|
def __getitem__(self, i):
|
|
return self.value.__getitem__(i)
|
|
|
|
def __setitem__(self, i, location):
|
|
_check_image_location(self.image_proxy.context,
|
|
self.image_proxy.store_api,
|
|
self.image_proxy.store_utils,
|
|
location)
|
|
location['status'] = 'active'
|
|
self.value.__setitem__(i, location)
|
|
_set_image_size(self.image_proxy.context,
|
|
self.image_proxy,
|
|
[location])
|
|
|
|
def __delitem__(self, i):
|
|
if isinstance(i, slice):
|
|
if i.step not in (None, 1):
|
|
raise NotImplementedError("slice with step")
|
|
self.__delslice__(i.start, i.stop)
|
|
return
|
|
location = None
|
|
try:
|
|
location = self.value[i]
|
|
except Exception:
|
|
del self.value[i]
|
|
return
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
del self.value[i]
|
|
|
|
def __delslice__(self, i, j):
|
|
i = 0 if i is None else max(i, 0)
|
|
j = len(self) if j is None else max(j, 0)
|
|
locations = []
|
|
try:
|
|
locations = self.value[i:j]
|
|
except Exception:
|
|
del self.value[i:j]
|
|
return
|
|
for location in locations:
|
|
self.image_proxy.store_utils.delete_image_location_from_backend(
|
|
self.image_proxy.context,
|
|
self.image_proxy.image.image_id,
|
|
location)
|
|
del self.value[i]
|
|
|
|
def __iadd__(self, other):
|
|
self.extend(other)
|
|
return self
|
|
|
|
def __contains__(self, location):
|
|
return location in self.value
|
|
|
|
def __len__(self):
|
|
return len(self.value)
|
|
|
|
def __cast(self, other):
|
|
if isinstance(other, StoreLocations):
|
|
return other.value
|
|
else:
|
|
return other
|
|
|
|
def __eq__(self, other):
|
|
return self.value == self.__cast(other)
|
|
|
|
def __lt__(self, other):
|
|
return self.value < self.__cast(other)
|
|
|
|
def __iter__(self):
|
|
return iter(self.value)
|
|
|
|
def __copy__(self):
|
|
return type(self)(self.image_proxy, self.value)
|
|
|
|
def __deepcopy__(self, memo):
|
|
# NOTE(zhiyan): Only copy location entries, others can be reused.
|
|
value = copy.deepcopy(self.value, memo)
|
|
self.image_proxy.image.locations = value
|
|
return type(self)(self.image_proxy, value)
|
|
|
|
|
|
def _locations_proxy(target, attr):
|
|
"""
|
|
Make a location property proxy on the image object.
|
|
|
|
:param target: the image object on which to add the proxy
|
|
:param attr: the property proxy we want to hook
|
|
"""
|
|
def get_attr(self):
|
|
value = getattr(getattr(self, target), attr)
|
|
return StoreLocations(self, value)
|
|
|
|
def set_attr(self, value):
|
|
if not isinstance(value, (list, StoreLocations)):
|
|
reason = _('Invalid locations')
|
|
raise exception.BadStoreUri(message=reason)
|
|
ori_value = getattr(getattr(self, target), attr)
|
|
if ori_value != value:
|
|
# NOTE(flwang): If all the URL of passed-in locations are same as
|
|
# current image locations, that means user would like to only
|
|
# update the metadata, not the URL.
|
|
ordered_value = sorted([loc['url'] for loc in value])
|
|
ordered_ori = sorted([loc['url'] for loc in ori_value])
|
|
if len(ori_value) > 0 and ordered_value != ordered_ori:
|
|
raise exception.Invalid(_('Original locations is not empty: '
|
|
'%s') % ori_value)
|
|
# NOTE(zhiyan): Check locations are all valid
|
|
# NOTE(flwang): If all the URL of passed-in locations are same as
|
|
# current image locations, then it's not necessary to verify those
|
|
# locations again. Otherwise, if there is any restricted scheme in
|
|
# existing locations. _check_image_location will fail.
|
|
if ordered_value != ordered_ori:
|
|
for loc in value:
|
|
_check_image_location(self.context,
|
|
self.store_api,
|
|
self.store_utils,
|
|
loc)
|
|
loc['status'] = 'active'
|
|
if _count_duplicated_locations(value, loc) > 1:
|
|
raise exception.DuplicateLocation(location=loc['url'])
|
|
_set_image_size(self.context, getattr(self, target), value)
|
|
else:
|
|
for loc in value:
|
|
loc['status'] = 'active'
|
|
return setattr(getattr(self, target), attr, list(value))
|
|
|
|
def del_attr(self):
|
|
value = getattr(getattr(self, target), attr)
|
|
while len(value):
|
|
self.store_utils.delete_image_location_from_backend(
|
|
self.context,
|
|
self.image.image_id,
|
|
value[0])
|
|
del value[0]
|
|
setattr(getattr(self, target), attr, value)
|
|
return delattr(getattr(self, target), attr)
|
|
|
|
return property(get_attr, set_attr, del_attr)
|
|
|
|
|
|
class ImageProxy(glance.domain.proxy.Image):
|
|
|
|
locations = _locations_proxy('image', 'locations')
|
|
|
|
def __init__(self, image, context, store_api, store_utils):
|
|
self.image = image
|
|
self.context = context
|
|
self.store_api = store_api
|
|
self.store_utils = store_utils
|
|
proxy_kwargs = {
|
|
'context': context,
|
|
'image': self,
|
|
'store_api': store_api,
|
|
}
|
|
super(ImageProxy, self).__init__(
|
|
image, member_repo_proxy_class=ImageMemberRepoProxy,
|
|
member_repo_proxy_kwargs=proxy_kwargs)
|
|
|
|
def delete(self):
|
|
self.image.delete()
|
|
if self.image.locations:
|
|
for location in self.image.locations:
|
|
self.store_utils.delete_image_location_from_backend(
|
|
self.context,
|
|
self.image.image_id,
|
|
location)
|
|
|
|
def _upload_to_store(self, data, verifier, store=None, size=None):
|
|
"""
|
|
Upload data to store
|
|
|
|
:param data: data to upload to store
|
|
:param verifier: for signature verification
|
|
:param store: store to upload data to
|
|
:param size: data size
|
|
:return:
|
|
"""
|
|
hashing_algo = self.image.os_hash_algo or CONF['hashing_algorithm']
|
|
if CONF.enabled_backends:
|
|
(location, size, checksum,
|
|
multihash, loc_meta) = self.store_api.add_with_multihash(
|
|
CONF,
|
|
self.image.image_id,
|
|
utils.LimitingReader(utils.CooperativeReader(data),
|
|
CONF.image_size_cap),
|
|
size,
|
|
store,
|
|
hashing_algo,
|
|
context=self.context,
|
|
verifier=verifier)
|
|
else:
|
|
(location,
|
|
size,
|
|
checksum,
|
|
multihash,
|
|
loc_meta) = self.store_api.add_to_backend_with_multihash(
|
|
CONF,
|
|
self.image.image_id,
|
|
utils.LimitingReader(utils.CooperativeReader(data),
|
|
CONF.image_size_cap),
|
|
size,
|
|
hashing_algo,
|
|
context=self.context,
|
|
verifier=verifier)
|
|
self._verify_signature(verifier, location, loc_meta)
|
|
for attr, data in {"size": size, "os_hash_value": multihash,
|
|
"checksum": checksum}.items():
|
|
self._verify_uploaded_data(data, attr)
|
|
self.image.locations.append({'url': location, 'metadata': loc_meta,
|
|
'status': 'active'})
|
|
self.image.checksum = checksum
|
|
self.image.os_hash_value = multihash
|
|
self.image.size = size
|
|
self.image.os_hash_algo = hashing_algo
|
|
|
|
def _verify_signature(self, verifier, location, loc_meta):
|
|
"""
|
|
Verify signature of uploaded data.
|
|
|
|
:param verifier: for signature verification
|
|
"""
|
|
# NOTE(bpoulos): if verification fails, exception will be raised
|
|
if verifier is not None:
|
|
try:
|
|
verifier.verify()
|
|
msg = _LI("Successfully verified signature for image %s")
|
|
LOG.info(msg, self.image.image_id)
|
|
except crypto_exception.InvalidSignature:
|
|
if CONF.enabled_backends:
|
|
self.store_api.delete(location,
|
|
loc_meta.get('store'),
|
|
context=self.context)
|
|
else:
|
|
self.store_api.delete_from_backend(location,
|
|
context=self.context)
|
|
raise cursive_exception.SignatureVerificationError(
|
|
_('Signature verification failed')
|
|
)
|
|
|
|
def _verify_uploaded_data(self, value, attribute_name):
|
|
"""
|
|
Verify value of attribute_name uploaded data
|
|
|
|
:param value: value to compare
|
|
:param attribute_name: attribute name of the image to compare with
|
|
"""
|
|
image_value = getattr(self.image, attribute_name)
|
|
if image_value is not None and value != image_value:
|
|
msg = _("%s of uploaded data is different from current "
|
|
"value set on the image.")
|
|
LOG.error(msg, attribute_name)
|
|
raise exception.UploadException(msg % attribute_name)
|
|
|
|
def set_data(self, data, size=None, backend=None, set_active=True):
|
|
if size is None:
|
|
size = 0 # NOTE(markwash): zero -> unknown size
|
|
|
|
# Create the verifier for signature verification (if correct properties
|
|
# are present)
|
|
extra_props = self.image.extra_properties
|
|
verifier = None
|
|
if signature_utils.should_create_verifier(extra_props):
|
|
# NOTE(bpoulos): if creating verifier fails, exception will be
|
|
# raised
|
|
img_signature = extra_props[signature_utils.SIGNATURE]
|
|
hash_method = extra_props[signature_utils.HASH_METHOD]
|
|
key_type = extra_props[signature_utils.KEY_TYPE]
|
|
cert_uuid = extra_props[signature_utils.CERT_UUID]
|
|
verifier = signature_utils.get_verifier(
|
|
context=self.context,
|
|
img_signature_certificate_uuid=cert_uuid,
|
|
img_signature_hash_method=hash_method,
|
|
img_signature=img_signature,
|
|
img_signature_key_type=key_type
|
|
)
|
|
|
|
self._upload_to_store(data, verifier, backend, size)
|
|
if set_active and self.image.status != 'active':
|
|
self.image.status = 'active'
|
|
|
|
def get_data(self, offset=0, chunk_size=None):
|
|
if not self.image.locations:
|
|
# NOTE(mclaren): This is the only set of arguments
|
|
# which work with this exception currently, see:
|
|
# https://bugs.launchpad.net/glance-store/+bug/1501443
|
|
# When the above glance_store bug is fixed we can
|
|
# add a msg as usual.
|
|
raise store.NotFound(image=None)
|
|
err = None
|
|
for loc in self.image.locations:
|
|
try:
|
|
backend = loc['metadata'].get('store')
|
|
if CONF.enabled_backends:
|
|
data, size = self.store_api.get(
|
|
loc['url'], backend, offset=offset,
|
|
chunk_size=chunk_size, context=self.context
|
|
)
|
|
else:
|
|
data, size = self.store_api.get_from_backend(
|
|
loc['url'],
|
|
offset=offset,
|
|
chunk_size=chunk_size,
|
|
context=self.context)
|
|
|
|
return data
|
|
except Exception as e:
|
|
LOG.warn(_LW('Get image %(id)s data failed: '
|
|
'%(err)s.'),
|
|
{'id': self.image.image_id,
|
|
'err': encodeutils.exception_to_unicode(e)})
|
|
err = e
|
|
# tried all locations
|
|
LOG.error(_LE('Glance tried all active locations to get data for '
|
|
'image %s but all have failed.'), self.image.image_id)
|
|
raise err
|
|
|
|
|
|
class ImageMemberRepoProxy(glance.domain.proxy.Repo):
|
|
def __init__(self, repo, image, context, store_api):
|
|
self.repo = repo
|
|
self.image = image
|
|
self.context = context
|
|
self.store_api = store_api
|
|
super(ImageMemberRepoProxy, self).__init__(repo)
|
|
|
|
def _set_acls(self):
|
|
public = self.image.visibility == 'public'
|
|
if self.image.locations and not public:
|
|
member_ids = [m.member_id for m in self.repo.list()]
|
|
for location in self.image.locations:
|
|
if CONF.enabled_backends:
|
|
self.store_api.set_acls_for_multi_store(
|
|
location['url'], location['metadata'].get('store'),
|
|
public=public, read_tenants=member_ids,
|
|
context=self.context
|
|
)
|
|
else:
|
|
self.store_api.set_acls(location['url'], public=public,
|
|
read_tenants=member_ids,
|
|
context=self.context)
|
|
|
|
def add(self, member):
|
|
super(ImageMemberRepoProxy, self).add(member)
|
|
self._set_acls()
|
|
|
|
def remove(self, member):
|
|
super(ImageMemberRepoProxy, self).remove(member)
|
|
self._set_acls()
|